rucio 32.8.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (481) hide show
  1. rucio/__init__.py +18 -0
  2. rucio/alembicrevision.py +16 -0
  3. rucio/api/__init__.py +14 -0
  4. rucio/api/account.py +266 -0
  5. rucio/api/account_limit.py +287 -0
  6. rucio/api/authentication.py +302 -0
  7. rucio/api/config.py +218 -0
  8. rucio/api/credential.py +60 -0
  9. rucio/api/did.py +726 -0
  10. rucio/api/dirac.py +71 -0
  11. rucio/api/exporter.py +60 -0
  12. rucio/api/heartbeat.py +62 -0
  13. rucio/api/identity.py +160 -0
  14. rucio/api/importer.py +46 -0
  15. rucio/api/lifetime_exception.py +95 -0
  16. rucio/api/lock.py +131 -0
  17. rucio/api/meta.py +85 -0
  18. rucio/api/permission.py +72 -0
  19. rucio/api/quarantined_replica.py +69 -0
  20. rucio/api/replica.py +528 -0
  21. rucio/api/request.py +220 -0
  22. rucio/api/rse.py +601 -0
  23. rucio/api/rule.py +335 -0
  24. rucio/api/scope.py +89 -0
  25. rucio/api/subscription.py +255 -0
  26. rucio/api/temporary_did.py +49 -0
  27. rucio/api/vo.py +112 -0
  28. rucio/client/__init__.py +16 -0
  29. rucio/client/accountclient.py +413 -0
  30. rucio/client/accountlimitclient.py +155 -0
  31. rucio/client/baseclient.py +929 -0
  32. rucio/client/client.py +77 -0
  33. rucio/client/configclient.py +113 -0
  34. rucio/client/credentialclient.py +54 -0
  35. rucio/client/didclient.py +691 -0
  36. rucio/client/diracclient.py +48 -0
  37. rucio/client/downloadclient.py +1674 -0
  38. rucio/client/exportclient.py +44 -0
  39. rucio/client/fileclient.py +51 -0
  40. rucio/client/importclient.py +42 -0
  41. rucio/client/lifetimeclient.py +74 -0
  42. rucio/client/lockclient.py +99 -0
  43. rucio/client/metaclient.py +137 -0
  44. rucio/client/pingclient.py +45 -0
  45. rucio/client/replicaclient.py +444 -0
  46. rucio/client/requestclient.py +109 -0
  47. rucio/client/rseclient.py +664 -0
  48. rucio/client/ruleclient.py +287 -0
  49. rucio/client/scopeclient.py +88 -0
  50. rucio/client/subscriptionclient.py +161 -0
  51. rucio/client/touchclient.py +78 -0
  52. rucio/client/uploadclient.py +871 -0
  53. rucio/common/__init__.py +14 -0
  54. rucio/common/cache.py +74 -0
  55. rucio/common/config.py +796 -0
  56. rucio/common/constants.py +92 -0
  57. rucio/common/constraints.py +18 -0
  58. rucio/common/didtype.py +187 -0
  59. rucio/common/dumper/__init__.py +306 -0
  60. rucio/common/dumper/consistency.py +449 -0
  61. rucio/common/dumper/data_models.py +325 -0
  62. rucio/common/dumper/path_parsing.py +65 -0
  63. rucio/common/exception.py +1092 -0
  64. rucio/common/extra.py +37 -0
  65. rucio/common/logging.py +404 -0
  66. rucio/common/pcache.py +1387 -0
  67. rucio/common/policy.py +84 -0
  68. rucio/common/schema/__init__.py +143 -0
  69. rucio/common/schema/atlas.py +411 -0
  70. rucio/common/schema/belleii.py +406 -0
  71. rucio/common/schema/cms.py +478 -0
  72. rucio/common/schema/domatpc.py +399 -0
  73. rucio/common/schema/escape.py +424 -0
  74. rucio/common/schema/generic.py +431 -0
  75. rucio/common/schema/generic_multi_vo.py +410 -0
  76. rucio/common/schema/icecube.py +404 -0
  77. rucio/common/schema/lsst.py +423 -0
  78. rucio/common/stomp_utils.py +160 -0
  79. rucio/common/stopwatch.py +56 -0
  80. rucio/common/test_rucio_server.py +148 -0
  81. rucio/common/types.py +158 -0
  82. rucio/common/utils.py +1946 -0
  83. rucio/core/__init__.py +14 -0
  84. rucio/core/account.py +426 -0
  85. rucio/core/account_counter.py +171 -0
  86. rucio/core/account_limit.py +357 -0
  87. rucio/core/authentication.py +563 -0
  88. rucio/core/config.py +386 -0
  89. rucio/core/credential.py +218 -0
  90. rucio/core/did.py +3102 -0
  91. rucio/core/did_meta_plugins/__init__.py +250 -0
  92. rucio/core/did_meta_plugins/did_column_meta.py +326 -0
  93. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +116 -0
  94. rucio/core/did_meta_plugins/filter_engine.py +573 -0
  95. rucio/core/did_meta_plugins/json_meta.py +215 -0
  96. rucio/core/did_meta_plugins/mongo_meta.py +199 -0
  97. rucio/core/did_meta_plugins/postgres_meta.py +317 -0
  98. rucio/core/dirac.py +208 -0
  99. rucio/core/distance.py +164 -0
  100. rucio/core/exporter.py +59 -0
  101. rucio/core/heartbeat.py +263 -0
  102. rucio/core/identity.py +290 -0
  103. rucio/core/importer.py +248 -0
  104. rucio/core/lifetime_exception.py +377 -0
  105. rucio/core/lock.py +474 -0
  106. rucio/core/message.py +241 -0
  107. rucio/core/meta.py +190 -0
  108. rucio/core/monitor.py +441 -0
  109. rucio/core/naming_convention.py +154 -0
  110. rucio/core/nongrid_trace.py +124 -0
  111. rucio/core/oidc.py +1339 -0
  112. rucio/core/permission/__init__.py +107 -0
  113. rucio/core/permission/atlas.py +1333 -0
  114. rucio/core/permission/belleii.py +1076 -0
  115. rucio/core/permission/cms.py +1166 -0
  116. rucio/core/permission/escape.py +1076 -0
  117. rucio/core/permission/generic.py +1128 -0
  118. rucio/core/permission/generic_multi_vo.py +1148 -0
  119. rucio/core/quarantined_replica.py +190 -0
  120. rucio/core/replica.py +3627 -0
  121. rucio/core/replica_sorter.py +368 -0
  122. rucio/core/request.py +2241 -0
  123. rucio/core/rse.py +1835 -0
  124. rucio/core/rse_counter.py +155 -0
  125. rucio/core/rse_expression_parser.py +460 -0
  126. rucio/core/rse_selector.py +277 -0
  127. rucio/core/rule.py +3419 -0
  128. rucio/core/rule_grouping.py +1473 -0
  129. rucio/core/scope.py +152 -0
  130. rucio/core/subscription.py +316 -0
  131. rucio/core/temporary_did.py +188 -0
  132. rucio/core/topology.py +448 -0
  133. rucio/core/trace.py +361 -0
  134. rucio/core/transfer.py +1233 -0
  135. rucio/core/vo.py +151 -0
  136. rucio/core/volatile_replica.py +123 -0
  137. rucio/daemons/__init__.py +14 -0
  138. rucio/daemons/abacus/__init__.py +14 -0
  139. rucio/daemons/abacus/account.py +106 -0
  140. rucio/daemons/abacus/collection_replica.py +113 -0
  141. rucio/daemons/abacus/rse.py +107 -0
  142. rucio/daemons/atropos/__init__.py +14 -0
  143. rucio/daemons/atropos/atropos.py +243 -0
  144. rucio/daemons/auditor/__init__.py +261 -0
  145. rucio/daemons/auditor/hdfs.py +86 -0
  146. rucio/daemons/auditor/srmdumps.py +284 -0
  147. rucio/daemons/automatix/__init__.py +14 -0
  148. rucio/daemons/automatix/automatix.py +281 -0
  149. rucio/daemons/badreplicas/__init__.py +14 -0
  150. rucio/daemons/badreplicas/minos.py +311 -0
  151. rucio/daemons/badreplicas/minos_temporary_expiration.py +173 -0
  152. rucio/daemons/badreplicas/necromancer.py +200 -0
  153. rucio/daemons/bb8/__init__.py +14 -0
  154. rucio/daemons/bb8/bb8.py +356 -0
  155. rucio/daemons/bb8/common.py +762 -0
  156. rucio/daemons/bb8/nuclei_background_rebalance.py +147 -0
  157. rucio/daemons/bb8/t2_background_rebalance.py +146 -0
  158. rucio/daemons/c3po/__init__.py +14 -0
  159. rucio/daemons/c3po/algorithms/__init__.py +14 -0
  160. rucio/daemons/c3po/algorithms/simple.py +131 -0
  161. rucio/daemons/c3po/algorithms/t2_free_space.py +125 -0
  162. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +127 -0
  163. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +279 -0
  164. rucio/daemons/c3po/c3po.py +342 -0
  165. rucio/daemons/c3po/collectors/__init__.py +14 -0
  166. rucio/daemons/c3po/collectors/agis.py +108 -0
  167. rucio/daemons/c3po/collectors/free_space.py +62 -0
  168. rucio/daemons/c3po/collectors/jedi_did.py +48 -0
  169. rucio/daemons/c3po/collectors/mock_did.py +46 -0
  170. rucio/daemons/c3po/collectors/network_metrics.py +63 -0
  171. rucio/daemons/c3po/collectors/workload.py +110 -0
  172. rucio/daemons/c3po/utils/__init__.py +14 -0
  173. rucio/daemons/c3po/utils/dataset_cache.py +40 -0
  174. rucio/daemons/c3po/utils/expiring_dataset_cache.py +45 -0
  175. rucio/daemons/c3po/utils/expiring_list.py +63 -0
  176. rucio/daemons/c3po/utils/popularity.py +82 -0
  177. rucio/daemons/c3po/utils/timeseries.py +76 -0
  178. rucio/daemons/cache/__init__.py +14 -0
  179. rucio/daemons/cache/consumer.py +191 -0
  180. rucio/daemons/common.py +391 -0
  181. rucio/daemons/conveyor/__init__.py +14 -0
  182. rucio/daemons/conveyor/common.py +530 -0
  183. rucio/daemons/conveyor/finisher.py +492 -0
  184. rucio/daemons/conveyor/poller.py +372 -0
  185. rucio/daemons/conveyor/preparer.py +198 -0
  186. rucio/daemons/conveyor/receiver.py +206 -0
  187. rucio/daemons/conveyor/stager.py +127 -0
  188. rucio/daemons/conveyor/submitter.py +379 -0
  189. rucio/daemons/conveyor/throttler.py +468 -0
  190. rucio/daemons/follower/__init__.py +14 -0
  191. rucio/daemons/follower/follower.py +97 -0
  192. rucio/daemons/hermes/__init__.py +14 -0
  193. rucio/daemons/hermes/hermes.py +738 -0
  194. rucio/daemons/judge/__init__.py +14 -0
  195. rucio/daemons/judge/cleaner.py +149 -0
  196. rucio/daemons/judge/evaluator.py +172 -0
  197. rucio/daemons/judge/injector.py +154 -0
  198. rucio/daemons/judge/repairer.py +144 -0
  199. rucio/daemons/oauthmanager/__init__.py +14 -0
  200. rucio/daemons/oauthmanager/oauthmanager.py +199 -0
  201. rucio/daemons/reaper/__init__.py +14 -0
  202. rucio/daemons/reaper/dark_reaper.py +272 -0
  203. rucio/daemons/reaper/light_reaper.py +255 -0
  204. rucio/daemons/reaper/reaper.py +701 -0
  205. rucio/daemons/replicarecoverer/__init__.py +14 -0
  206. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +487 -0
  207. rucio/daemons/storage/__init__.py +14 -0
  208. rucio/daemons/storage/consistency/__init__.py +14 -0
  209. rucio/daemons/storage/consistency/actions.py +753 -0
  210. rucio/daemons/tracer/__init__.py +14 -0
  211. rucio/daemons/tracer/kronos.py +513 -0
  212. rucio/daemons/transmogrifier/__init__.py +14 -0
  213. rucio/daemons/transmogrifier/transmogrifier.py +753 -0
  214. rucio/daemons/undertaker/__init__.py +14 -0
  215. rucio/daemons/undertaker/undertaker.py +137 -0
  216. rucio/db/__init__.py +14 -0
  217. rucio/db/sqla/__init__.py +38 -0
  218. rucio/db/sqla/constants.py +192 -0
  219. rucio/db/sqla/migrate_repo/__init__.py +14 -0
  220. rucio/db/sqla/migrate_repo/env.py +111 -0
  221. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +71 -0
  222. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +50 -0
  223. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +61 -0
  224. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +46 -0
  225. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +93 -0
  226. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +78 -0
  227. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +46 -0
  228. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +53 -0
  229. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +69 -0
  230. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +42 -0
  231. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +46 -0
  232. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +61 -0
  233. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +42 -0
  234. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +141 -0
  235. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +75 -0
  236. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +75 -0
  237. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +46 -0
  238. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +51 -0
  239. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +135 -0
  240. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +65 -0
  241. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +42 -0
  242. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +66 -0
  243. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +54 -0
  244. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +43 -0
  245. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +46 -0
  246. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +47 -0
  247. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +54 -0
  248. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +39 -0
  249. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +48 -0
  250. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +47 -0
  251. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +48 -0
  252. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +59 -0
  253. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +47 -0
  254. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +72 -0
  255. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +46 -0
  256. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +45 -0
  257. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +48 -0
  258. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +48 -0
  259. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +42 -0
  260. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +69 -0
  261. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +46 -0
  262. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +78 -0
  263. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +62 -0
  264. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +74 -0
  265. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +44 -0
  266. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +67 -0
  267. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +134 -0
  268. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +58 -0
  269. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +79 -0
  270. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +61 -0
  271. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +45 -0
  272. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +46 -0
  273. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +65 -0
  274. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +42 -0
  275. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +46 -0
  276. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +46 -0
  277. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +80 -0
  278. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +43 -0
  279. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +61 -0
  280. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +47 -0
  281. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +46 -0
  282. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +52 -0
  283. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +42 -0
  284. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +65 -0
  285. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +46 -0
  286. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +47 -0
  287. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +45 -0
  288. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +46 -0
  289. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +48 -0
  290. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +50 -0
  291. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +59 -0
  292. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +48 -0
  293. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +108 -0
  294. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +57 -0
  295. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +51 -0
  296. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +50 -0
  297. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +46 -0
  298. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +42 -0
  299. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +93 -0
  300. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +73 -0
  301. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +52 -0
  302. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +45 -0
  303. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +46 -0
  304. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +54 -0
  305. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +48 -0
  306. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +70 -0
  307. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +48 -0
  308. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +95 -0
  309. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +55 -0
  310. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +74 -0
  311. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +78 -0
  312. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +49 -0
  313. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +124 -0
  314. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +60 -0
  315. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +53 -0
  316. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +56 -0
  317. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +67 -0
  318. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +50 -0
  319. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +46 -0
  320. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +92 -0
  321. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +42 -0
  322. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +46 -0
  323. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +147 -0
  324. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +78 -0
  325. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +53 -0
  326. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +74 -0
  327. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +56 -0
  328. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +46 -0
  329. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +68 -0
  330. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +48 -0
  331. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +149 -0
  332. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +106 -0
  333. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +47 -0
  334. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +45 -0
  335. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +105 -0
  336. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +52 -0
  337. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +106 -0
  338. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +30 -0
  339. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +75 -0
  340. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +49 -0
  341. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +45 -0
  342. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +38 -0
  343. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +44 -0
  344. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +46 -0
  345. rucio/db/sqla/models.py +1834 -0
  346. rucio/db/sqla/sautils.py +48 -0
  347. rucio/db/sqla/session.py +470 -0
  348. rucio/db/sqla/types.py +207 -0
  349. rucio/db/sqla/util.py +521 -0
  350. rucio/rse/__init__.py +97 -0
  351. rucio/rse/protocols/__init__.py +14 -0
  352. rucio/rse/protocols/cache.py +123 -0
  353. rucio/rse/protocols/dummy.py +112 -0
  354. rucio/rse/protocols/gfal.py +701 -0
  355. rucio/rse/protocols/globus.py +243 -0
  356. rucio/rse/protocols/gsiftp.py +93 -0
  357. rucio/rse/protocols/http_cache.py +83 -0
  358. rucio/rse/protocols/mock.py +124 -0
  359. rucio/rse/protocols/ngarc.py +210 -0
  360. rucio/rse/protocols/posix.py +251 -0
  361. rucio/rse/protocols/protocol.py +530 -0
  362. rucio/rse/protocols/rclone.py +365 -0
  363. rucio/rse/protocols/rfio.py +137 -0
  364. rucio/rse/protocols/srm.py +339 -0
  365. rucio/rse/protocols/ssh.py +414 -0
  366. rucio/rse/protocols/storm.py +207 -0
  367. rucio/rse/protocols/webdav.py +547 -0
  368. rucio/rse/protocols/xrootd.py +295 -0
  369. rucio/rse/rsemanager.py +752 -0
  370. rucio/tests/__init__.py +14 -0
  371. rucio/tests/common.py +244 -0
  372. rucio/tests/common_server.py +132 -0
  373. rucio/transfertool/__init__.py +14 -0
  374. rucio/transfertool/fts3.py +1484 -0
  375. rucio/transfertool/globus.py +200 -0
  376. rucio/transfertool/globus_library.py +182 -0
  377. rucio/transfertool/mock.py +81 -0
  378. rucio/transfertool/transfertool.py +212 -0
  379. rucio/vcsversion.py +11 -0
  380. rucio/version.py +46 -0
  381. rucio/web/__init__.py +14 -0
  382. rucio/web/rest/__init__.py +14 -0
  383. rucio/web/rest/flaskapi/__init__.py +14 -0
  384. rucio/web/rest/flaskapi/authenticated_bp.py +28 -0
  385. rucio/web/rest/flaskapi/v1/__init__.py +14 -0
  386. rucio/web/rest/flaskapi/v1/accountlimits.py +234 -0
  387. rucio/web/rest/flaskapi/v1/accounts.py +1088 -0
  388. rucio/web/rest/flaskapi/v1/archives.py +100 -0
  389. rucio/web/rest/flaskapi/v1/auth.py +1642 -0
  390. rucio/web/rest/flaskapi/v1/common.py +385 -0
  391. rucio/web/rest/flaskapi/v1/config.py +305 -0
  392. rucio/web/rest/flaskapi/v1/credentials.py +213 -0
  393. rucio/web/rest/flaskapi/v1/dids.py +2204 -0
  394. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  395. rucio/web/rest/flaskapi/v1/export.py +77 -0
  396. rucio/web/rest/flaskapi/v1/heartbeats.py +129 -0
  397. rucio/web/rest/flaskapi/v1/identities.py +263 -0
  398. rucio/web/rest/flaskapi/v1/import.py +133 -0
  399. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +315 -0
  400. rucio/web/rest/flaskapi/v1/locks.py +360 -0
  401. rucio/web/rest/flaskapi/v1/main.py +83 -0
  402. rucio/web/rest/flaskapi/v1/meta.py +226 -0
  403. rucio/web/rest/flaskapi/v1/metrics.py +37 -0
  404. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  405. rucio/web/rest/flaskapi/v1/ping.py +89 -0
  406. rucio/web/rest/flaskapi/v1/redirect.py +366 -0
  407. rucio/web/rest/flaskapi/v1/replicas.py +1866 -0
  408. rucio/web/rest/flaskapi/v1/requests.py +841 -0
  409. rucio/web/rest/flaskapi/v1/rses.py +2204 -0
  410. rucio/web/rest/flaskapi/v1/rules.py +824 -0
  411. rucio/web/rest/flaskapi/v1/scopes.py +161 -0
  412. rucio/web/rest/flaskapi/v1/subscriptions.py +646 -0
  413. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  414. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  415. rucio/web/rest/flaskapi/v1/tmp_dids.py +115 -0
  416. rucio/web/rest/flaskapi/v1/traces.py +100 -0
  417. rucio/web/rest/flaskapi/v1/vos.py +280 -0
  418. rucio/web/rest/main.py +19 -0
  419. rucio/web/rest/metrics.py +28 -0
  420. rucio-32.8.6.data/data/rucio/etc/alembic.ini.template +71 -0
  421. rucio-32.8.6.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  422. rucio-32.8.6.data/data/rucio/etc/globus-config.yml.template +5 -0
  423. rucio-32.8.6.data/data/rucio/etc/ldap.cfg.template +30 -0
  424. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  425. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  426. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  427. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  428. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  429. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  430. rucio-32.8.6.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  431. rucio-32.8.6.data/data/rucio/etc/rucio.cfg.atlas.client.template +42 -0
  432. rucio-32.8.6.data/data/rucio/etc/rucio.cfg.template +257 -0
  433. rucio-32.8.6.data/data/rucio/etc/rucio_multi_vo.cfg.template +234 -0
  434. rucio-32.8.6.data/data/rucio/requirements.txt +55 -0
  435. rucio-32.8.6.data/data/rucio/tools/bootstrap.py +34 -0
  436. rucio-32.8.6.data/data/rucio/tools/merge_rucio_configs.py +147 -0
  437. rucio-32.8.6.data/data/rucio/tools/reset_database.py +40 -0
  438. rucio-32.8.6.data/scripts/rucio +2540 -0
  439. rucio-32.8.6.data/scripts/rucio-abacus-account +75 -0
  440. rucio-32.8.6.data/scripts/rucio-abacus-collection-replica +47 -0
  441. rucio-32.8.6.data/scripts/rucio-abacus-rse +79 -0
  442. rucio-32.8.6.data/scripts/rucio-admin +2434 -0
  443. rucio-32.8.6.data/scripts/rucio-atropos +61 -0
  444. rucio-32.8.6.data/scripts/rucio-auditor +199 -0
  445. rucio-32.8.6.data/scripts/rucio-automatix +51 -0
  446. rucio-32.8.6.data/scripts/rucio-bb8 +58 -0
  447. rucio-32.8.6.data/scripts/rucio-c3po +86 -0
  448. rucio-32.8.6.data/scripts/rucio-cache-client +135 -0
  449. rucio-32.8.6.data/scripts/rucio-cache-consumer +43 -0
  450. rucio-32.8.6.data/scripts/rucio-conveyor-finisher +59 -0
  451. rucio-32.8.6.data/scripts/rucio-conveyor-poller +67 -0
  452. rucio-32.8.6.data/scripts/rucio-conveyor-preparer +38 -0
  453. rucio-32.8.6.data/scripts/rucio-conveyor-receiver +44 -0
  454. rucio-32.8.6.data/scripts/rucio-conveyor-stager +77 -0
  455. rucio-32.8.6.data/scripts/rucio-conveyor-submitter +140 -0
  456. rucio-32.8.6.data/scripts/rucio-conveyor-throttler +105 -0
  457. rucio-32.8.6.data/scripts/rucio-dark-reaper +54 -0
  458. rucio-32.8.6.data/scripts/rucio-dumper +159 -0
  459. rucio-32.8.6.data/scripts/rucio-follower +45 -0
  460. rucio-32.8.6.data/scripts/rucio-hermes +55 -0
  461. rucio-32.8.6.data/scripts/rucio-judge-cleaner +90 -0
  462. rucio-32.8.6.data/scripts/rucio-judge-evaluator +138 -0
  463. rucio-32.8.6.data/scripts/rucio-judge-injector +45 -0
  464. rucio-32.8.6.data/scripts/rucio-judge-repairer +45 -0
  465. rucio-32.8.6.data/scripts/rucio-kronos +45 -0
  466. rucio-32.8.6.data/scripts/rucio-light-reaper +53 -0
  467. rucio-32.8.6.data/scripts/rucio-minos +54 -0
  468. rucio-32.8.6.data/scripts/rucio-minos-temporary-expiration +51 -0
  469. rucio-32.8.6.data/scripts/rucio-necromancer +121 -0
  470. rucio-32.8.6.data/scripts/rucio-oauth-manager +64 -0
  471. rucio-32.8.6.data/scripts/rucio-reaper +84 -0
  472. rucio-32.8.6.data/scripts/rucio-replica-recoverer +249 -0
  473. rucio-32.8.6.data/scripts/rucio-storage-consistency-actions +75 -0
  474. rucio-32.8.6.data/scripts/rucio-transmogrifier +78 -0
  475. rucio-32.8.6.data/scripts/rucio-undertaker +77 -0
  476. rucio-32.8.6.dist-info/METADATA +83 -0
  477. rucio-32.8.6.dist-info/RECORD +481 -0
  478. rucio-32.8.6.dist-info/WHEEL +5 -0
  479. rucio-32.8.6.dist-info/licenses/AUTHORS.rst +94 -0
  480. rucio-32.8.6.dist-info/licenses/LICENSE +201 -0
  481. rucio-32.8.6.dist-info/top_level.txt +1 -0
rucio/core/scope.py ADDED
@@ -0,0 +1,152 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from re import match
17
+ from traceback import format_exc
18
+ from typing import TYPE_CHECKING
19
+
20
+ from sqlalchemy.exc import IntegrityError
21
+
22
+ from rucio.common.exception import AccountNotFound, Duplicate, RucioException, VONotFound
23
+ from rucio.core.vo import vo_exists
24
+ from rucio.db.sqla import models
25
+ from rucio.db.sqla.constants import AccountStatus, ScopeStatus
26
+ from rucio.db.sqla.session import read_session, transactional_session
27
+
28
+ if TYPE_CHECKING:
29
+ from sqlalchemy.orm import Session
30
+
31
+
32
+ @transactional_session
33
+ def add_scope(scope, account, *, session: "Session"):
34
+ """ add a scope for the given account name.
35
+
36
+ :param scope: the name for the new scope.
37
+ :param account: the account to add the scope to.
38
+ :param session: The database session in use.
39
+ """
40
+
41
+ if not vo_exists(vo=scope.vo, session=session):
42
+ raise VONotFound('VO {} not found'.format(scope.vo))
43
+
44
+ result = session.query(models.Account).filter_by(account=account, status=AccountStatus.ACTIVE).first()
45
+ if result is None:
46
+ raise AccountNotFound('Account ID \'%s\' does not exist' % account)
47
+
48
+ new_scope = models.Scope(scope=scope, account=account, status=ScopeStatus.OPEN)
49
+ try:
50
+ new_scope.save(session=session)
51
+ except IntegrityError as e:
52
+ if match('.*IntegrityError.*ORA-00001: unique constraint.*SCOPES_PK.*violated.*', e.args[0]) \
53
+ or match('.*IntegrityError.*Duplicate entry.*for key.*', e.args[0]) \
54
+ or match('.*IntegrityError.*UNIQUE constraint failed: scopes.scope.*', e.args[0]) \
55
+ or match('.*IntegrityError.*duplicate key value violates unique constraint.*', e.args[0]) \
56
+ or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', e.args[0]) \
57
+ or match('.*IntegrityError.*columns? .*not unique.*', e.args[0]):
58
+ raise Duplicate('Scope \'%s\' already exists!' % scope)
59
+ else:
60
+ raise RucioException(e)
61
+ except:
62
+ raise RucioException(str(format_exc()))
63
+
64
+
65
+ @read_session
66
+ def bulk_add_scopes(scopes, account, skipExisting=False, *, session: "Session"):
67
+ """ add a group of scopes, this call should not be exposed to users.
68
+
69
+ :param scopes: a list of scopes to be added.
70
+ :param account: the account associated to the scopes.
71
+ :param session: The database session in use.
72
+ """
73
+
74
+ for scope in scopes:
75
+ try:
76
+ add_scope(scope, account, session=session)
77
+ except Duplicate:
78
+ if not skipExisting:
79
+ raise
80
+
81
+
82
+ @read_session
83
+ def list_scopes(filter_={}, *, session: "Session"):
84
+ """
85
+ Lists all scopes.
86
+ :param filter_: Dictionary of attributes by which the input data should be filtered
87
+ :param session: The database session in use.
88
+
89
+ :returns: A list containing all scopes.
90
+ """
91
+ scope_list = []
92
+ query = session.query(models.Scope).filter(models.Scope.status != ScopeStatus.DELETED)
93
+ for filter_type in filter_:
94
+ if filter_type == 'scope':
95
+ if '*' in filter_['scope'].internal:
96
+ scope_str = filter_['scope'].internal.replace('*', '%')
97
+ query = query.filter(models.Scope.scope.like(scope_str))
98
+ else:
99
+ query = query.filter_by(scope=filter_['scope'])
100
+
101
+ for s in query:
102
+ scope_list.append(s.scope)
103
+ return scope_list
104
+
105
+
106
+ @read_session
107
+ def get_scopes(account, *, session: "Session"):
108
+ """ get all scopes defined for an account.
109
+
110
+ :param account: the account name to list the scopes of.
111
+ :param session: The database session in use.
112
+
113
+ :returns: a list of all scope names for this account.
114
+ """
115
+
116
+ result = session.query(models.Account).filter_by(account=account).first()
117
+
118
+ if result is None:
119
+ raise AccountNotFound('Account ID \'%s\' does not exist' % account)
120
+
121
+ scope_list = []
122
+
123
+ for s in session.query(models.Scope).filter_by(account=account).filter(models.Scope.status != ScopeStatus.DELETED):
124
+ scope_list.append(s.scope)
125
+
126
+ return scope_list
127
+
128
+
129
+ @read_session
130
+ def check_scope(scope_to_check, *, session: "Session"):
131
+ """ check to see if scope exists.
132
+
133
+ :param scope: the scope to check.
134
+ :param session: The database session in use.
135
+
136
+ :returns: True or false
137
+ """
138
+
139
+ return True if session.query(models.Scope).filter_by(scope=scope_to_check).first() else False
140
+
141
+
142
+ @read_session
143
+ def is_scope_owner(scope, account, *, session: "Session"):
144
+ """ check to see if account owns the scope.
145
+
146
+ :param scope: the scope to check.
147
+ :param account: the account to check.
148
+ :param session: The database session in use.
149
+
150
+ :returns: True or false
151
+ """
152
+ return True if session.query(models.Scope).filter_by(scope=scope, account=account).first() else False
@@ -0,0 +1,316 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ import datetime
17
+ import logging
18
+ import re
19
+ from configparser import NoOptionError, NoSectionError
20
+ from json import dumps
21
+ from typing import TYPE_CHECKING
22
+
23
+ from sqlalchemy import func
24
+ from sqlalchemy.exc import IntegrityError, StatementError
25
+ from sqlalchemy.orm import aliased
26
+ from sqlalchemy.orm.exc import NoResultFound
27
+
28
+ from rucio.common.config import config_get
29
+ from rucio.common.exception import SubscriptionNotFound, SubscriptionDuplicate, RucioException
30
+ from rucio.db.sqla import models
31
+ from rucio.db.sqla.constants import SubscriptionState
32
+ from rucio.db.sqla.session import transactional_session, stream_session, read_session
33
+
34
+ if TYPE_CHECKING:
35
+ from collections.abc import Callable, Iterator
36
+ from typing import Any, Optional
37
+ from sqlalchemy.orm import Session
38
+ from rucio.common.types import InternalAccount
39
+ LoggerFunction = Callable[..., Any]
40
+ SubscriptionType = dict
41
+
42
+
43
+ @transactional_session
44
+ def add_subscription(name: str,
45
+ account: "InternalAccount",
46
+ filter_: str,
47
+ replication_rules: str,
48
+ comments: str,
49
+ lifetime: "Optional[int]" = None,
50
+ retroactive: "Optional[bool]" = False,
51
+ dry_run: "Optional[bool]" = False,
52
+ priority: "Optional[int]" = 3,
53
+ *, session: "Session") -> str:
54
+ """
55
+ Adds a new subscription which will be verified against every new added file and dataset
56
+
57
+ :param account: Account identifier
58
+ :type account: String
59
+ :param name: Name of the subscription
60
+ :type name: String
61
+ :param filter_: Dictionary of attributes by which the input data should be filtered
62
+ **Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
63
+ :type filter_: Dict
64
+ :param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
65
+ :type replication_rules: Dict
66
+ :param comments: Comments for the subscription
67
+ :type comments: String
68
+ :param lifetime: Subscription's lifetime (days)
69
+ :type lifetime: Integer or None
70
+ :param retroactive: Flag to know if the subscription should be applied on previous data
71
+ :type retroactive: Boolean
72
+ :param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
73
+ :type dry_run: Boolean
74
+ :param priority: The priority of the subscription
75
+ :type priority: Integer
76
+ :param session: The database session in use.
77
+
78
+ :returns: The subscriptionid
79
+ """
80
+ try:
81
+ keep_history = config_get('subscriptions', 'keep_history')
82
+ except (NoOptionError, NoSectionError, RuntimeError):
83
+ keep_history = False
84
+
85
+ SubscriptionHistory = models.SubscriptionHistory
86
+ retroactive = bool(retroactive) # Force boolean type, necessary for strict SQL
87
+ state = SubscriptionState.ACTIVE
88
+ if retroactive:
89
+ state = SubscriptionState.NEW
90
+ if lifetime:
91
+ date_lifetime = datetime.datetime.utcnow() + datetime.timedelta(days=lifetime)
92
+ else:
93
+ date_lifetime = None
94
+ new_subscription = models.Subscription(name=name,
95
+ filter=filter_,
96
+ account=account,
97
+ replication_rules=replication_rules,
98
+ state=state,
99
+ lifetime=date_lifetime,
100
+ retroactive=retroactive,
101
+ policyid=priority, comments=comments)
102
+ if keep_history:
103
+ subscription_history = SubscriptionHistory(id=new_subscription.id,
104
+ name=new_subscription.name,
105
+ filter=new_subscription.filter,
106
+ account=new_subscription.account,
107
+ replication_rules=new_subscription.replication_rules,
108
+ state=new_subscription.state,
109
+ lifetime=new_subscription.lifetime,
110
+ retroactive=new_subscription.retroactive,
111
+ policyid=new_subscription.policyid,
112
+ comments=new_subscription.comments)
113
+ try:
114
+ new_subscription.save(session=session)
115
+ if keep_history:
116
+ subscription_history.save(session=session)
117
+ except IntegrityError as error:
118
+ if re.match('.*IntegrityError.*ORA-00001: unique constraint.*SUBSCRIPTIONS_PK.*violated.*', error.args[0])\
119
+ or re.match(".*IntegrityError.*UNIQUE constraint failed: subscriptions.name, subscriptions.account.*", error.args[0])\
120
+ or re.match('.*IntegrityError.*columns? name.*account.*not unique.*', error.args[0]) \
121
+ or re.match('.*IntegrityError.*ORA-00001: unique constraint.*SUBSCRIPTIONS_NAME_ACCOUNT_UQ.*violated.*', error.args[0])\
122
+ or re.match('.*IntegrityError.*1062.*Duplicate entry.*', error.args[0]) \
123
+ or re.match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
124
+ or re.match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]):
125
+ raise SubscriptionDuplicate('Subscription \'%s\' owned by \'%s\' already exists!' % (name, account))
126
+ raise RucioException(error.args)
127
+ return str(new_subscription.id)
128
+
129
+
130
+ @transactional_session
131
+ def update_subscription(name: str,
132
+ account: "InternalAccount",
133
+ metadata: dict,
134
+ *, session: "Session") -> None:
135
+ """
136
+ Updates a subscription
137
+
138
+ :param name: Name of the subscription
139
+ :type name: String
140
+ :param account: Account identifier
141
+ :type account: String
142
+ :param metadata: Dictionary of metadata to update. Supported keys : filter, replication_rules, comments, lifetime, retroactive, dry_run, priority, last_processed
143
+ :type metadata: Dict
144
+ :param session: The database session in use.
145
+
146
+ :raises: SubscriptionNotFound if subscription is not found
147
+ """
148
+ try:
149
+ keep_history = config_get('subscriptions', 'keep_history')
150
+ except (NoOptionError, NoSectionError, RuntimeError):
151
+ keep_history = False
152
+ values = {'state': SubscriptionState.UPDATED}
153
+ if 'filter' in metadata and metadata['filter']:
154
+ values['filter'] = dumps(metadata['filter'])
155
+ if 'replication_rules' in metadata and metadata['replication_rules']:
156
+ values['replication_rules'] = dumps(metadata['replication_rules'])
157
+ if 'lifetime' in metadata and metadata['lifetime']:
158
+ values['lifetime'] = datetime.datetime.utcnow() + datetime.timedelta(days=float(metadata['lifetime']))
159
+ if 'retroactive' in metadata and metadata['retroactive']:
160
+ values['retroactive'] = metadata['retroactive']
161
+ if 'dry_run' in metadata and metadata['dry_run']:
162
+ values['dry_run'] = metadata['dry_run']
163
+ if 'comments' in metadata and metadata['comments']:
164
+ values['comments'] = metadata['comments']
165
+ if 'priority' in metadata and metadata['priority']:
166
+ values['policyid'] = metadata['priority']
167
+ if 'last_processed' in metadata and metadata['last_processed']:
168
+ values['last_processed'] = metadata['last_processed']
169
+ if 'state' in metadata and metadata['state'] == SubscriptionState.INACTIVE:
170
+ values['state'] = SubscriptionState.INACTIVE
171
+ values['expired_at'] = datetime.datetime.utcnow()
172
+
173
+ SubscriptionHistory = models.SubscriptionHistory
174
+ try:
175
+ subscription = session.query(models.Subscription).filter_by(account=account, name=name).one()
176
+
177
+ # To avoid update in the subscription history table whenever last processed field is changed
178
+ current_subscription_state = subscription.to_dict()
179
+ new_subscription_state = values.copy()
180
+
181
+ for key in ["updated_at", "last_processed"]:
182
+ new_subscription_state.pop(key, "None")
183
+ current_subscription_state.pop(key, "None")
184
+
185
+ subscription.update(values)
186
+ if keep_history and current_subscription_state != new_subscription_state:
187
+ subscription_history = SubscriptionHistory(id=subscription.id,
188
+ name=subscription.name,
189
+ filter=subscription.filter,
190
+ account=subscription.account,
191
+ replication_rules=subscription.replication_rules,
192
+ state=subscription.state,
193
+ lifetime=subscription.lifetime,
194
+ retroactive=subscription.retroactive,
195
+ policyid=subscription.policyid,
196
+ comments=subscription.comments,
197
+ last_processed=subscription.last_processed,
198
+ expired_at=subscription.expired_at,
199
+ updated_at=subscription.updated_at,
200
+ created_at=subscription.created_at)
201
+ subscription_history.save(session=session)
202
+ except NoResultFound:
203
+ raise SubscriptionNotFound(f"Subscription for account '{account}' named '{name}' not found")
204
+
205
+
206
+ @stream_session
207
+ def list_subscriptions(name: "Optional[str]" = None,
208
+ account: "Optional[InternalAccount]" = None,
209
+ state: "Optional[SubscriptionState]" = None,
210
+ *, session: "Session",
211
+ logger: "LoggerFunction" = logging.log) -> "Iterator[SubscriptionType]":
212
+ """
213
+ Returns a dictionary with the subscription information :
214
+ Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
215
+
216
+ :param name: Name of the subscription
217
+ :type name: String
218
+ :param account: Account identifier
219
+ :type account: String
220
+ :param session: The database session in use.
221
+ :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
222
+
223
+ :returns: Dictionary containing subscription parameter
224
+ :rtype: Dict
225
+ :raises: exception.NotFound if subscription is not found
226
+ """
227
+ query = session.query(models.Subscription)
228
+ try:
229
+ if name:
230
+ query = query.filter_by(name=name)
231
+ if account:
232
+ if '*' in account.internal:
233
+ account_str = account.internal.replace('*', '%')
234
+ query = query.filter(models.Subscription.account.like(account_str))
235
+ else:
236
+ query = query.filter_by(account=account)
237
+ if state:
238
+ query = query.filter_by(state=state)
239
+ except IntegrityError as error:
240
+ logger(logging.ERROR, str(error))
241
+ raise RucioException(error.args)
242
+ found = False
243
+ for row in query:
244
+ found = True
245
+ yield row.to_dict()
246
+ if not found:
247
+ raise SubscriptionNotFound(f"Subscription for account '{account}' named '{name}' not found")
248
+
249
+
250
+ @transactional_session
251
+ def delete_subscription(subscription_id: str, *, session: "Session") -> None:
252
+ """
253
+ Deletes a subscription
254
+
255
+ :param subscription_id: Subscription identifier
256
+ :type subscription_id: String
257
+ """
258
+ session.query(models.Subscription).filter_by(id=subscription_id).delete()
259
+
260
+
261
+ @stream_session
262
+ def list_subscription_rule_states(name=None, account=None, *, session: "Session", logger=logging.log):
263
+ """Returns a list of with the number of rules per state for a subscription.
264
+
265
+ :param name: Name of the subscription
266
+ :param account: Account identifier
267
+ :param session: The database session in use.
268
+ :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
269
+ :returns: List with tuple (account, name, state, count)
270
+ """
271
+ subscription = aliased(models.Subscription)
272
+ rule = aliased(models.ReplicationRule)
273
+ # count needs a label to allow conversion to dict (label name can be changed)
274
+ query = session.query(
275
+ subscription.account, subscription.name, rule.state, func.count().label('count')).join(
276
+ rule, subscription.id == rule.subscription_id)
277
+
278
+ try:
279
+ if name:
280
+ query = query.filter(subscription.name == name)
281
+
282
+ if account:
283
+ if '*' in account.internal:
284
+ account_str = account.internal.replace('*', '%')
285
+ query = query.filter(subscription.account.like(account_str))
286
+ else:
287
+ query = query.filter(subscription.account == account)
288
+
289
+ except IntegrityError as error:
290
+ logger(logging.ERROR, str(error))
291
+ raise RucioException(error.args)
292
+
293
+ query = query.group_by(subscription.account, subscription.name, rule.state)
294
+
295
+ for row in query:
296
+ yield row
297
+
298
+
299
+ @read_session
300
+ def get_subscription_by_id(subscription_id, *, session: "Session"):
301
+ """
302
+ Get a specific subscription by id.
303
+
304
+ :param subscription_id: The subscription_id to select.
305
+ :param session: The database session in use.
306
+
307
+ :raises: SubscriptionNotFound if no Subscription can be found.
308
+ """
309
+
310
+ try:
311
+ subscription = session.query(models.Subscription).filter_by(id=subscription_id).one()
312
+ return subscription.to_dict()
313
+ except NoResultFound:
314
+ raise SubscriptionNotFound('No subscription with the id %s found' % (subscription_id))
315
+ except StatementError:
316
+ raise RucioException('Badly formatted subscription id (%s)' % (subscription_id))
@@ -0,0 +1,188 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from datetime import datetime
17
+ from typing import TYPE_CHECKING
18
+
19
+ from sqlalchemy import and_, or_, func, delete
20
+ from sqlalchemy.sql.expression import case, insert, update
21
+
22
+ from rucio.core.did import attach_dids
23
+ from rucio.core.replica import add_replica
24
+ from rucio.db.sqla import models, filter_thread_work
25
+ from rucio.db.sqla.session import read_session, transactional_session
26
+
27
+ # from rucio.rse import rsemanager as rsemgr
28
+
29
+ if TYPE_CHECKING:
30
+ from sqlalchemy.orm import Session
31
+
32
+
33
+ @transactional_session
34
+ def add_temporary_dids(dids, account, *, session: "Session"):
35
+ """
36
+ Bulk add temporary data identifiers.
37
+
38
+ :param dids: A list of dids.
39
+ :param account: The account owner.
40
+ :param session: The database session in use.
41
+ """
42
+ temporary_dids = []
43
+ for did in dids:
44
+
45
+ if did.get('pfn'):
46
+ did['path'] = did['pfn']
47
+ # In waiting to properly extract the path
48
+ # p = rsemgr.create_protocol(rse_settings=rsemgr.get_rse_info(rse, session=session), operation='write', scheme=scheme)
49
+ # if not replica_rse.deterministic:
50
+ # pfns = p.parse_pfns(pfns=pfns)
51
+ # tmp = pfns[file['pfn']]
52
+ # file['path'] = ''.join([tmp['path'], tmp['name']])
53
+
54
+ temporary_dids.append({'scope': did['scope'],
55
+ 'name': did['name'],
56
+ 'rse_id': did['rse_id'],
57
+ 'path': did.get('path'),
58
+ 'bytes': did.get('bytes'),
59
+ 'md5': did.get('md5'),
60
+ 'adler32': did.get('adler32'),
61
+ 'guid': did.get('guid'),
62
+ 'events': did.get('envents'),
63
+ 'parent_scope': did.get('parent_scope'),
64
+ 'parent_name': did.get('parent_name'),
65
+ 'offset': did.get('offset'),
66
+ 'expired_at': datetime.utcnow()})
67
+ try:
68
+ if temporary_dids:
69
+ session.execute(insert(models.TemporaryDataIdentifier), temporary_dids)
70
+ except:
71
+ raise
72
+
73
+
74
+ @transactional_session
75
+ def compose(scope, name, rse_id, bytes_, sources, account,
76
+ md5=None, adler32=None, pfn=None, meta={}, rules=[],
77
+ parent_scope=None, parent_name=None,
78
+ *, session: "Session"):
79
+ """
80
+ Concatenates a list of existing dids into a new file replica
81
+
82
+ :param scope: the scope name.
83
+ :param name: The data identifier name.
84
+ :param rse_id: the rse id.
85
+ :param bytes_: the size of the file.
86
+ :sources sources: The list of temporary DIDs.
87
+ :param account: The account owner.
88
+ :param md5: The md5 checksum.
89
+ :param adler32: The adler32 checksum.
90
+ :param pfn: Physical file name (for nondeterministic rse).
91
+ :param meta: Meta-data associated with the file. Represented as key/value pairs in a dictionary.
92
+ :param rules: Replication rules associated with the file. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
93
+ :param parent_scope: Possible dataset scope.
94
+ :param parent_name: Possibe dataset name.
95
+ :param session: The database session in use.
96
+ """
97
+ # Create the new file did and replica
98
+ add_replica(rse_id=rse_id, scope=scope, name=name, bytes_=bytes_, account=account,
99
+ adler32=adler32, md5=md5, pfn=pfn, meta=meta, rules=rules,
100
+ session=session)
101
+
102
+ # Attach the file to a dataset
103
+ if parent_scope and parent_name:
104
+ attach_dids(scope=parent_scope, name=parent_name,
105
+ dids=[{'scope': scope, 'name': name}], account=account,
106
+ rse_id=None, session=session)
107
+
108
+ # Mark the merged dids as obsolete
109
+ now, expired_dids = datetime.utcnow(), []
110
+ for source in sources:
111
+ expired_dids.append({'scope': source['scope'],
112
+ 'name': source['name'],
113
+ 'expired_at': now})
114
+ if expired_dids:
115
+ session.execute(update(models.TemporaryDataIdentifier), expired_dids)
116
+
117
+
118
+ @read_session
119
+ def list_expired_temporary_dids(rse_id, limit, worker_number=None, total_workers=None,
120
+ *, session: "Session"):
121
+ """
122
+ List expired temporary DIDs.
123
+
124
+ :param rse_id: the rse id.
125
+ :param limit: The maximum number of replicas returned.
126
+ :param worker_number: id of the executing worker.
127
+ :param total_workers: Number of total workers.
128
+ :param session: The database session in use.
129
+
130
+ :returns: a list of dictionary replica.
131
+ """
132
+ is_none = None
133
+ query = session.query(models.TemporaryDataIdentifier.scope,
134
+ models.TemporaryDataIdentifier.name,
135
+ models.TemporaryDataIdentifier.path,
136
+ models.TemporaryDataIdentifier.bytes).\
137
+ with_hint(models.TemporaryDataIdentifier, "INDEX(tmp_dids TMP_DIDS_EXPIRED_AT_IDX)", 'oracle').\
138
+ filter(case((models.TemporaryDataIdentifier.expired_at != is_none, models.TemporaryDataIdentifier.rse_id)) == rse_id)
139
+
140
+ query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
141
+
142
+ return [{'path': path,
143
+ 'rse_id': rse_id,
144
+ 'scope': scope,
145
+ 'name': name,
146
+ 'bytes': bytes_}
147
+ for scope, name, path, bytes_ in query.limit(limit)]
148
+
149
+
150
+ @transactional_session
151
+ def delete_temporary_dids(dids, *, session: "Session"):
152
+ """
153
+ Delete temporary file replicas.
154
+
155
+ :param dids: the list of files to delete.
156
+ :param session
157
+ """
158
+ where_clause = []
159
+ for did in dids:
160
+ where_clause.append(and_(models.TemporaryDataIdentifier.scope == did['scope'],
161
+ models.TemporaryDataIdentifier.name == did['name']))
162
+
163
+ if where_clause:
164
+ stmt = delete(models.TemporaryDataIdentifier).\
165
+ prefix_with("/*+ INDEX(tmp_dids TMP_DIDS_PK) */", dialect='oracle').\
166
+ where(or_(*where_clause)).execution_options(synchronize_session=False)
167
+ result = session.execute(stmt)
168
+ return result.rowcount
169
+ return
170
+
171
+
172
+ @read_session
173
+ def get_count_of_expired_temporary_dids(rse_id, *, session: "Session"):
174
+ """
175
+ List expired temporary DIDs.
176
+
177
+ :param rse_id: the rse id.
178
+ :param session: The database session in use.
179
+
180
+ :returns: a count number.
181
+ """
182
+ is_none = None
183
+ count = session.query(func.count(models.TemporaryDataIdentifier.scope)).\
184
+ with_hint(models.TemporaryDataIdentifier, "INDEX(tmp_dids TMP_DIDS_EXPIRED_AT_IDX)", 'oracle').\
185
+ filter(case((models.TemporaryDataIdentifier.expired_at != is_none, models.TemporaryDataIdentifier.rse_id)) == rse_id).\
186
+ one()
187
+
188
+ return count[0] or 0