rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,27 +17,37 @@ import glob
18
17
  import logging
19
18
  import os
20
19
  import queue as Queue
21
- from datetime import datetime
22
- from datetime import timedelta
23
-
24
20
  import select
21
+ from datetime import datetime, timedelta
22
+ from typing import TYPE_CHECKING, Optional
25
23
 
26
24
  from rucio.common import config
27
- from rucio.common.dumper import LogPipeHandler
28
- from rucio.common.dumper import mkdir
29
- from rucio.common.dumper import temp_file
25
+ from rucio.common.dumper import LogPipeHandler, mkdir, temp_file
30
26
  from rucio.common.dumper.consistency import Consistency
31
27
  from rucio.common.types import InternalAccount, InternalScope
32
28
  from rucio.common.utils import chunks
33
29
  from rucio.core.quarantined_replica import add_quarantined_replicas
34
30
  from rucio.core.replica import declare_bad_file_replicas, list_replicas
35
- from rucio.core.rse import get_rse_usage, get_rse_id
31
+ from rucio.core.rse import get_rse_id, get_rse_usage
36
32
  from rucio.daemons.auditor import srmdumps
37
33
  from rucio.daemons.auditor.hdfs import ReplicaFromHDFS
38
34
  from rucio.db.sqla.constants import BadFilesStatus
39
35
 
40
-
41
- def consistency(rse, delta, configuration, cache_dir, results_dir):
36
+ if TYPE_CHECKING:
37
+ from collections.abc import Iterable
38
+ from configparser import RawConfigParser
39
+ from multiprocessing import Queue as QueueType
40
+ from multiprocessing.connection import Connection
41
+ from multiprocessing.synchronize import Event
42
+
43
+
44
+ def consistency(
45
+ rse: str,
46
+ delta: timedelta,
47
+ configuration: "RawConfigParser",
48
+ cache_dir: str,
49
+ results_dir: str
50
+ ) -> Optional[str]:
42
51
  logger = logging.getLogger('auditor-worker')
43
52
  rsedump, rsedate = srmdumps.download_rse_dump(rse, configuration, destdir=cache_dir)
44
53
  results_path = os.path.join(results_dir, '{0}_{1}'.format(rse, rsedate.strftime('%Y%m%d'))) # pylint: disable=no-member
@@ -66,11 +75,12 @@ def consistency(rse, delta, configuration, cache_dir, results_dir):
66
75
  return results_path
67
76
 
68
77
 
69
- def guess_replica_info(path):
78
+ def guess_replica_info(
79
+ path: str
80
+ ) -> tuple[Optional[str], str]:
70
81
  """Try to extract the scope and name from a path.
71
82
 
72
- ``path`` should be an ``str`` with the relative path to the file on
73
- the RSE.
83
+ ``path``: relative path to the file on the RSE.
74
84
 
75
85
  Returns a ``tuple`` of which the first element is the scope of the
76
86
  replica and the second element is its name.
@@ -84,7 +94,10 @@ def guess_replica_info(path):
84
94
  return items[0], items[-1]
85
95
 
86
96
 
87
- def bz2_compress_file(source, chunk_size=65000):
97
+ def bz2_compress_file(
98
+ source: str,
99
+ chunk_size: int = 65000
100
+ ) -> str:
88
101
  """Compress a file with bzip2.
89
102
 
90
103
  The destination is the path passed through ``source`` extended with
@@ -93,13 +106,11 @@ def bz2_compress_file(source, chunk_size=65000):
93
106
  Errors are deliberately not handled gracefully. Any exceptions
94
107
  should be propagated to the caller.
95
108
 
96
- ``source`` should be an ``str`` with the absolute path to the file
97
- to compress.
109
+ ``source``: absolute path to the file to compress.
98
110
 
99
- ``chunk_size`` should be an ``int`` with the size (in bytes) of the
100
- chunks by which to read the file.
111
+ ``chunk_size``: size (in bytes) of the chunks by which to read the file.
101
112
 
102
- Returns an ``str`` with the destination path.
113
+ Returns the destination path.
103
114
  """
104
115
  destination = '{}.bz2'.format(source)
105
116
  with open(source) as plain, bz2.BZ2File(destination, 'w') as compressed:
@@ -112,14 +123,18 @@ def bz2_compress_file(source, chunk_size=65000):
112
123
  return destination
113
124
 
114
125
 
115
- def process_output(output, sanity_check=True, compress=True):
126
+ def process_output(
127
+ output: str,
128
+ sanity_check: bool = True,
129
+ compress: bool = True
130
+ ) -> None:
116
131
  """Perform post-consistency-check actions.
117
132
 
118
133
  DARK files are put in the quarantined-replica table so that they
119
134
  may be deleted by the Dark Reaper. LOST files are reported as
120
135
  suspicious so that they may be further checked by the cloud squads.
121
136
 
122
- ``output`` should be an ``str`` with the absolute path to the file
137
+ ``output``: absolute path to the file
123
138
  produced by ``consistency()``. It must maintain its naming
124
139
  convention.
125
140
 
@@ -155,7 +170,7 @@ def process_output(output, sanity_check=True, compress=True):
155
170
  rse = os.path.basename(output[:output.rfind('_')])
156
171
  rse_id = get_rse_id(rse=rse)
157
172
  usage = get_rse_usage(rse_id=rse_id, source='rucio')[0]
158
- threshold = config.config_get_float('auditor', 'threshold', False, 0.2)
173
+ threshold = config.config_get_float('auditor', 'threshold', False, 0.1)
159
174
 
160
175
  # Perform a basic sanity check by comparing the number of entries
161
176
  # with the total number of files on the RSE. If the percentage is
@@ -190,7 +205,16 @@ def process_output(output, sanity_check=True, compress=True):
190
205
  logger.debug('Compressed "%s"', destination)
191
206
 
192
207
 
193
- def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps, delta_in_days):
208
+ def check(
209
+ queue: "QueueType",
210
+ retry: "QueueType",
211
+ terminate: "Event",
212
+ logpipe: "Connection",
213
+ cache_dir: str,
214
+ results_dir: str,
215
+ keep_dumps: bool,
216
+ delta_in_days: int
217
+ ) -> None:
194
218
  logger = logging.getLogger('auditor-worker')
195
219
  lib_logger = logging.getLogger('dumper')
196
220
 
@@ -213,7 +237,7 @@ def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps,
213
237
 
214
238
  while not terminate.is_set():
215
239
  try:
216
- rse, attemps = queue.get(timeout=30)
240
+ rse, attempts = queue.get(timeout=30)
217
241
  except Queue.Empty:
218
242
  continue
219
243
  start = datetime.now()
@@ -225,7 +249,7 @@ def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps,
225
249
  process_output(output)
226
250
  except:
227
251
  elapsed = (datetime.now() - start).total_seconds() / 60
228
- logger.error('Check of "%s" failed in %d minutes, %d remaining attemps', rse, elapsed, attemps, exc_info=True)
252
+ logger.error('Check of "%s" failed in %d minutes, %d remaining attempts', rse, elapsed, attempts, exc_info=True)
229
253
  success = False
230
254
  else:
231
255
  elapsed = (datetime.now() - start).total_seconds() / 60
@@ -239,11 +263,15 @@ def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps,
239
263
  for fil in remove:
240
264
  os.remove(fil)
241
265
 
242
- if not success and attemps > 0:
243
- retry.put((rse, attemps - 1))
266
+ if not success and attempts > 0:
267
+ retry.put((rse, attempts - 1))
244
268
 
245
269
 
246
- def activity_logger(logpipes, logfilename, terminate):
270
+ def activity_logger(
271
+ logpipes: "Iterable[Connection]",
272
+ logfilename: str,
273
+ terminate: "Event"
274
+ ) -> None:
247
275
  handler = logging.handlers.RotatingFileHandler(
248
276
  logfilename,
249
277
  maxBytes=20971520,
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,13 +19,19 @@ import re
20
19
  import shutil
21
20
  import subprocess
22
21
  import tempfile
22
+ from typing import TYPE_CHECKING
23
23
 
24
- from rucio.common.dumper import DUMPS_CACHE_DIR
25
- from rucio.common.dumper import temp_file
24
+ from rucio.common.dumper import DUMPS_CACHE_DIR, temp_file
26
25
  from rucio.common.dumper.data_models import Replica
27
26
 
27
+ if TYPE_CHECKING:
28
+ from datetime import datetime
28
29
 
29
- def _hdfs_get(src_url, dst_path):
30
+
31
+ def _hdfs_get(
32
+ src_url: str,
33
+ dst_path: str
34
+ ) -> None:
30
35
  cmd = ['hadoop', 'fs', '-get', src_url, dst_path]
31
36
  get = subprocess.Popen(
32
37
  cmd,
@@ -34,7 +39,7 @@ def _hdfs_get(src_url, dst_path):
34
39
  )
35
40
  _, stderr = get.communicate()
36
41
  if get.returncode != 0:
37
- raise IOError('_hdfs_get(): "{0}": {1}. Return code {2}'.format(
42
+ raise OSError('_hdfs_get(): "{0}": {1}. Return code {2}'.format(
38
43
  ' '.join(cmd),
39
44
  stderr,
40
45
  get.returncode,
@@ -45,7 +50,13 @@ class ReplicaFromHDFS(Replica):
45
50
  BASE_URL = '/user/rucio01/reports/{0}/replicas_per_rse/{1}/*'
46
51
 
47
52
  @classmethod
48
- def download(cls, rse, date, cache_dir=DUMPS_CACHE_DIR, buffer_size=65536):
53
+ def download(
54
+ cls,
55
+ rse: str,
56
+ date: "datetime",
57
+ cache_dir: str = DUMPS_CACHE_DIR,
58
+ buffer_size: int = 65536
59
+ ) -> str:
49
60
  logger = logging.getLogger('auditor.hdfs')
50
61
 
51
62
  if not os.path.isdir(cache_dir):
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,32 +21,53 @@ import logging
22
21
  import operator
23
22
  import os
24
23
  import re
24
+ from typing import IO, TYPE_CHECKING, Any, Optional
25
25
 
26
26
  import gfal2
27
27
  import requests
28
28
 
29
29
  from rucio.common.config import get_config_dirs
30
- from rucio.common.dumper import DUMPS_CACHE_DIR
31
- from rucio.common.dumper import http_download_to_file, gfal_download_to_file, ddmendpoint_url, temp_file
30
+ from rucio.common.constants import RseAttr
31
+ from rucio.common.dumper import DUMPS_CACHE_DIR, HTTPDownloadFailed, ddmendpoint_url, gfal_download_to_file, http_download_to_file, temp_file
32
+ from rucio.core.credential import get_signed_url
33
+ from rucio.core.rse import get_rse_id, list_rse_attributes
34
+
35
+ if TYPE_CHECKING:
36
+ from collections.abc import Iterable
32
37
 
33
38
  CHUNK_SIZE = 10485760
34
39
 
35
- __DUMPERCONFIGDIRS = (os.path.join(confdir, 'auditor') for confdir in get_config_dirs())
36
- __DUMPERCONFIGDIRS = list(filter(os.path.exists, __DUMPERCONFIGDIRS))
40
+ __DUMPERCONFIGDIRS = list(
41
+ filter(
42
+ os.path.exists,
43
+ (
44
+ os.path.join(confdir, 'auditor') for confdir in get_config_dirs()
45
+ )
46
+ )
47
+ )
48
+
49
+ OBJECTSTORE_NUM_TRIES = 30
37
50
 
38
51
 
39
- class Parser(ConfigParser.RawConfigParser, object):
52
+ class Parser(ConfigParser.RawConfigParser):
40
53
  '''
41
54
  RawConfigParser subclass that doesn't modify the the name of the options
42
- and removes any quotes arround the string values.
55
+ and removes any quotes around the string values.
43
56
  '''
44
57
  remove_quotes_re = re.compile(r"^'(.+)'$")
45
58
  remove_double_quotes_re = re.compile(r'^"(.+)"$')
46
59
 
47
- def optionxform(self, optionstr):
60
+ def optionxform(
61
+ self,
62
+ optionstr: str
63
+ ) -> str:
48
64
  return optionstr
49
65
 
50
- def get(self, section, option):
66
+ def get(
67
+ self,
68
+ section: str,
69
+ option: str
70
+ ) -> Any:
51
71
  value = super(Parser, self).get(section, option)
52
72
  if isinstance(value, str):
53
73
  value = self.remove_quotes_re.sub(r'\1', value)
@@ -58,18 +78,23 @@ class Parser(ConfigParser.RawConfigParser, object):
58
78
  return [(name, self.get(section, name)) for name in self.options(section)]
59
79
 
60
80
 
61
- def mkdir(dir_):
81
+ def mkdir(dir_: str) -> None:
62
82
  '''
63
83
  This functions creates the `dir` directory if it doesn't exist. If `dir`
64
84
  already exists this function does nothing.
65
85
  '''
66
86
  try:
67
87
  os.mkdir(dir_)
68
- except OSError as e:
69
- assert e.errno == 17
88
+ except OSError as error:
89
+ if error.errno != 17:
90
+ raise error
70
91
 
71
92
 
72
- def get_newest(base_url, url_pattern, links):
93
+ def get_newest(
94
+ base_url: str,
95
+ url_pattern: str,
96
+ links: "Iterable[str]"
97
+ ) -> tuple[str, datetime.datetime]:
73
98
  '''
74
99
  Returns a tuple with the newest url in the `links` list matching the
75
100
  pattern `url_pattern` and a datetime object representing the creation
@@ -104,7 +129,7 @@ def get_newest(base_url, url_pattern, links):
104
129
  return max(times, key=operator.itemgetter(1))
105
130
 
106
131
 
107
- def gfal_links(base_url):
132
+ def gfal_links(base_url: str) -> list[str]:
108
133
  '''
109
134
  Returns a list of the urls contained in `base_url`.
110
135
  '''
@@ -112,19 +137,22 @@ def gfal_links(base_url):
112
137
  return ['/'.join((base_url, f)) for f in ctxt.listdir(str(base_url))]
113
138
 
114
139
 
115
- class _LinkCollector(HTMLParser.HTMLParser, object):
140
+ class _LinkCollector(HTMLParser.HTMLParser):
116
141
  def __init__(self):
117
142
  super(_LinkCollector, self).__init__()
118
143
  self.links = []
119
144
 
120
- def handle_starttag(self, tag, attrs):
145
+ def handle_starttag(
146
+ self, tag: str,
147
+ attrs: "Iterable[tuple[str, str]]"
148
+ ) -> None:
121
149
  if tag == 'a':
122
150
  self.links.append(
123
151
  next(value for key, value in attrs if key == 'href')
124
152
  )
125
153
 
126
154
 
127
- def http_links(base_url):
155
+ def http_links(base_url: str) -> list[str]:
128
156
  '''
129
157
  Returns a list of the urls contained in `base_url`.
130
158
  '''
@@ -169,7 +197,7 @@ protocol_funcs = {
169
197
  }
170
198
 
171
199
 
172
- def protocol(url):
200
+ def protocol(url: str) -> str:
173
201
  '''
174
202
  Given the URL `url` returns a string with the protocol part.
175
203
  '''
@@ -180,25 +208,26 @@ def protocol(url):
180
208
  return proto
181
209
 
182
210
 
183
- def get_links(base_url):
211
+ def get_links(base_url: str) -> list[str]:
184
212
  '''
185
213
  Given the URL `base_url` returns the URLs linked or contained in it.
186
214
  '''
187
215
  return protocol_funcs[protocol(base_url)]['links'](base_url)
188
216
 
189
217
 
190
- def download(url, filename):
218
+ def download(url: str, filename: IO) -> None:
191
219
  '''
192
220
  Given the URL `url` downloads its contents on `filename`.
193
221
  '''
194
222
  return protocol_funcs[protocol(url)]['download'](url, filename)
195
223
 
196
224
 
197
- def parse_configuration(conf_dirs=__DUMPERCONFIGDIRS):
225
+ def parse_configuration(conf_dirs: Optional[list[str]] = None) -> Parser:
198
226
  '''
199
227
  Parses the configuration for the endpoints contained in `conf_dir`.
200
228
  Returns a ConfParser.RawConfParser subclass instance.
201
229
  '''
230
+ conf_dirs = conf_dirs or __DUMPERCONFIGDIRS
202
231
  logger = logging.getLogger('auditor.srmdumps')
203
232
  if len(conf_dirs) == 0:
204
233
  logger.error('No configuration directory given to load SRM dumps paths')
@@ -213,18 +242,23 @@ def parse_configuration(conf_dirs=__DUMPERCONFIGDIRS):
213
242
  return configuration
214
243
 
215
244
 
216
- def download_rse_dump(rse, configuration, date='latest', destdir=DUMPS_CACHE_DIR):
245
+ def download_rse_dump(
246
+ rse: str,
247
+ configuration: ConfigParser.RawConfigParser,
248
+ date: Optional[datetime.datetime] = None,
249
+ destdir: str = DUMPS_CACHE_DIR
250
+ ) -> tuple[str, datetime.datetime]:
217
251
  '''
218
252
  Downloads the dump for the given ddmendpoint. If this endpoint does not
219
- follow the standarized method to publish the dumps it should have an
253
+ follow the standardized method to publish the dumps it should have an
220
254
  entry in the `configuration` object describing how to download the dump.
221
255
 
222
256
  `rse` is the DDMEndpoint name.
223
257
 
224
258
  `configuration` is a RawConfigParser subclass.
225
259
 
226
- `date` is a datetime instance with the date of the desired dump or 'latest'
227
- to download the lastest available dump.
260
+ `date` is a datetime instance with the date of the desired dump or None
261
+ to download the latest available dump.
228
262
 
229
263
  `destdir` is the directory where the dump will be saved (the final component
230
264
  in the path is created if it doesn't exist).
@@ -234,34 +268,71 @@ def download_rse_dump(rse, configuration, date='latest', destdir=DUMPS_CACHE_DIR
234
268
  '''
235
269
  logger = logging.getLogger('auditor.srmdumps')
236
270
  base_url, url_pattern = generate_url(rse, configuration)
237
- if date == 'latest':
238
- logger.debug('Looking for site dumps in: "%s"', base_url)
239
- links = get_links(base_url)
240
- url, date = get_newest(base_url, url_pattern, links)
241
- else:
242
- url = '{0}/{1}'.format(base_url, date.strftime(url_pattern))
243
271
 
244
272
  if not os.path.isdir(destdir):
245
273
  os.mkdir(destdir)
246
274
 
247
- filename = '{0}_{1}_{2}_{3}'.format(
248
- 'ddmendpoint',
249
- rse,
250
- date.strftime('%d-%m-%Y'),
251
- hashlib.sha1(url.encode()).hexdigest()
252
- )
253
- filename = re.sub(r'\W', '-', filename)
254
- path = os.path.join(destdir, filename)
255
-
256
- if not os.path.exists(path):
257
- logger.debug('Trying to download: "%s"', url)
258
- with temp_file(destdir, final_name=filename) as (f, _):
259
- download(url, f)
275
+ # check for objectstores, which need to be handled differently
276
+ rse_id = get_rse_id(rse)
277
+ rse_attr = list_rse_attributes(rse_id)
278
+ if RseAttr.IS_OBJECT_STORE in rse_attr and rse_attr[RseAttr.IS_OBJECT_STORE] is not False:
279
+ tries = 1
280
+ if date is None:
281
+ # on objectstores, can't list dump files, so try the last N dates
282
+ date = datetime.datetime.now()
283
+ tries = OBJECTSTORE_NUM_TRIES
284
+ path = ''
285
+ while tries > 0:
286
+ url = '{0}/{1}'.format(base_url, date.strftime(url_pattern))
287
+
288
+ filename = '{0}_{1}_{2}_{3}'.format(
289
+ 'ddmendpoint',
290
+ rse,
291
+ date.strftime('%d-%m-%Y'),
292
+ hashlib.sha1(url.encode()).hexdigest()
293
+ )
294
+ filename = re.sub(r'\W', '-', filename)
295
+ path = os.path.join(destdir, filename)
296
+ if not os.path.exists(path):
297
+ logger.debug('Trying to download: "%s"', url)
298
+ if RseAttr.SIGN_URL in rse_attr:
299
+ url = get_signed_url(rse_id, rse_attr[RseAttr.SIGN_URL], 'read', url)
300
+ try:
301
+ with temp_file(destdir, final_name=filename) as (f, _):
302
+ download(url, f)
303
+ tries = 0
304
+ except (HTTPDownloadFailed, gfal2.GError):
305
+ tries -= 1
306
+ date = date - datetime.timedelta(1)
307
+ else:
308
+ if date is None:
309
+ logger.debug('Looking for site dumps in: "%s"', base_url)
310
+ links = get_links(base_url)
311
+ url, date = get_newest(base_url, url_pattern, links)
312
+ else:
313
+ url = '{0}/{1}'.format(base_url, date.strftime(url_pattern))
314
+
315
+ filename = '{0}_{1}_{2}_{3}'.format(
316
+ 'ddmendpoint',
317
+ rse,
318
+ date.strftime('%d-%m-%Y'),
319
+ hashlib.sha1(url.encode()).hexdigest()
320
+ )
321
+ filename = re.sub(r'\W', '-', filename)
322
+ path = os.path.join(destdir, filename)
323
+
324
+ if not os.path.exists(path):
325
+ logger.debug('Trying to download: "%s"', url)
326
+ with temp_file(destdir, final_name=filename) as (f, _):
327
+ download(url, f)
260
328
 
261
329
  return (path, date)
262
330
 
263
331
 
264
- def generate_url(rse, config):
332
+ def generate_url(
333
+ rse: str,
334
+ config: ConfigParser.RawConfigParser
335
+ ) -> tuple[str, str]:
265
336
  '''
266
337
  :param rse: Name of the endpoint.
267
338
  :param config: RawConfigParser instance which may have configuration
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,40 +15,42 @@
16
15
  import functools
17
16
  import logging
18
17
  import random
18
+ import secrets
19
19
  import tempfile
20
20
  import threading
21
21
  from configparser import NoOptionError, NoSectionError
22
22
  from datetime import datetime
23
23
  from json import load
24
24
  from os import remove, rmdir
25
- from typing import TYPE_CHECKING
25
+ from typing import TYPE_CHECKING, Any, Optional
26
26
 
27
27
  import rucio.db.sqla.util
28
28
  from rucio.client import Client
29
29
  from rucio.client.uploadclient import UploadClient
30
30
  from rucio.common import exception
31
- from rucio.common.config import config_get, config_get_int, config_get_bool
31
+ from rucio.common.config import config_get, config_get_bool, config_get_int
32
32
  from rucio.common.logging import setup_logging
33
33
  from rucio.common.stopwatch import Stopwatch
34
- from rucio.common.types import InternalScope
34
+ from rucio.common.types import InternalScope, LoggerFunction
35
35
  from rucio.common.utils import execute, generate_uuid
36
36
  from rucio.core.monitor import MetricManager
37
37
  from rucio.core.scope import list_scopes
38
38
  from rucio.core.vo import map_vo
39
- from rucio.daemons.common import run_daemon
39
+ from rucio.daemons.common import HeartbeatHandler, run_daemon
40
40
 
41
41
  if TYPE_CHECKING:
42
+ from collections.abc import Mapping
42
43
  from types import FrameType
43
- from typing import Optional
44
44
 
45
- from rucio.daemons.common import HeartbeatHandler
46
45
 
47
46
  METRICS = MetricManager(module=__name__)
48
47
  graceful_stop = threading.Event()
49
48
  DAEMON_NAME = "automatix"
50
49
 
51
50
 
52
- def get_data_distribution(inputfile: str):
51
+ def get_data_distribution(
52
+ inputfile: str
53
+ ) -> tuple[dict[str, int], dict[str, Any]]:
53
54
  with open(inputfile) as data_file:
54
55
  data = load(data_file)
55
56
  probabilities = {}
@@ -62,8 +63,11 @@ def get_data_distribution(inputfile: str):
62
63
  return probabilities, data
63
64
 
64
65
 
65
- def choose_element(probabilities: dict, data: str) -> float:
66
- rnd = random.uniform(0, 1)
66
+ def choose_element(
67
+ probabilities: "Mapping[str, int]",
68
+ data: "Mapping[str, Any]"
69
+ ) -> dict[str, Any]:
70
+ rnd = random.uniform(0, 1) # noqa: S311
67
71
  prob = 0
68
72
  for key in probabilities:
69
73
  prob = probabilities[key]
@@ -72,7 +76,11 @@ def choose_element(probabilities: dict, data: str) -> float:
72
76
  return data[key]
73
77
 
74
78
 
75
- def generate_file(fname: str, size: int, logger=logging.log) -> int:
79
+ def generate_file(
80
+ fname: str,
81
+ size: int,
82
+ logger: LoggerFunction = logging.log
83
+ ) -> int:
76
84
  cmd = "/bin/dd if=/dev/urandom of=%s bs=%s count=1" % (fname, size)
77
85
  exitcode, out, err = execute(cmd)
78
86
  logger(logging.DEBUG, out)
@@ -80,7 +88,11 @@ def generate_file(fname: str, size: int, logger=logging.log) -> int:
80
88
  return exitcode
81
89
 
82
90
 
83
- def generate_didname(metadata: dict, dsn: str, did_type: str) -> str:
91
+ def generate_didname(
92
+ metadata: "Mapping[str, str]",
93
+ dsn: Optional[str],
94
+ did_type: str
95
+ ) -> str:
84
96
  try:
85
97
  did_prefix = config_get("automatix", "did_prefix")
86
98
  except (NoOptionError, NoSectionError, RuntimeError):
@@ -102,11 +114,11 @@ def generate_didname(metadata: dict, dsn: str, did_type: str) -> str:
102
114
  elif field == "uuid":
103
115
  field_str = generate_uuid()
104
116
  elif field == "randint":
105
- field_str = str(random.randint(0, 100000))
117
+ field_str = str(secrets.choice(range(0, 100001)))
106
118
  else:
107
119
  field_str = metadata.get(field, None)
108
120
  if not field_str:
109
- field_str = str(random.randint(0, 100000))
121
+ field_str = str(secrets.choice(range(0, 100001)))
110
122
  file_name = "%s%s%s" % (file_name, separator, field_str)
111
123
  len_separator = len(separator)
112
124
  return file_name[len_separator:]
@@ -133,7 +145,7 @@ def automatix(inputfile: str, sleep_time: int, once: bool = False) -> None:
133
145
  )
134
146
 
135
147
 
136
- def run_once(heartbeat_handler: "HeartbeatHandler", inputfile: str, **_kwargs) -> bool:
148
+ def run_once(heartbeat_handler: HeartbeatHandler, inputfile: str, **_kwargs) -> bool:
137
149
 
138
150
  _, _, logger = heartbeat_handler.live()
139
151
  try:
@@ -166,7 +178,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", inputfile: str, **_kwargs) -
166
178
  account = config_get("automatix", "account", raise_exception=False, default="root")
167
179
  scope = config_get("automatix", "scope", raise_exception=False, default="test")
168
180
  client = Client(account=account)
169
- vo = map_vo(client.vo)
181
+ vo = map_vo(client.vo) # type: ignore
170
182
  filters = {"scope": InternalScope("*", vo=vo)}
171
183
  scopes = list_scopes(filter_=filters)
172
184
  if InternalScope(scope, vo=vo) not in scopes:
@@ -230,7 +242,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", inputfile: str, **_kwargs) -
230
242
  upload_client = UploadClient(client)
231
243
  ret = upload_client.upload(files)
232
244
  if ret == 0:
233
- logger(logging.INFO, "%s sucessfully registered" % dsn)
245
+ logger(logging.INFO, "%s successfully registered" % dsn)
234
246
  METRICS.counter(name="addnewdataset.done").inc()
235
247
  METRICS.counter(name="addnewfile.done").inc(nbfiles)
236
248
  METRICS.timer(name='datasetinjection').observe(stopwatch.elapsed)
@@ -274,7 +286,7 @@ def run(
274
286
  [thread.join(timeout=3.14) for thread in threads]
275
287
 
276
288
 
277
- def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
289
+ def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
278
290
  """
279
291
  Graceful exit.
280
292
  """
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");