rucio 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (266) hide show
  1. rucio/alembicrevision.py +1 -1
  2. rucio/{daemons/c3po/collectors → cli}/__init__.py +1 -0
  3. rucio/cli/account.py +216 -0
  4. rucio-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
  5. rucio-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
  6. rucio/cli/command.py +272 -0
  7. rucio/cli/config.py +72 -0
  8. rucio/cli/did.py +191 -0
  9. rucio/cli/download.py +128 -0
  10. rucio/cli/lifetime_exception.py +33 -0
  11. rucio/cli/replica.py +162 -0
  12. rucio/cli/rse.py +293 -0
  13. rucio/cli/rule.py +158 -0
  14. rucio/cli/scope.py +40 -0
  15. rucio/cli/subscription.py +73 -0
  16. rucio/cli/upload.py +60 -0
  17. rucio/cli/utils.py +226 -0
  18. rucio/client/accountclient.py +0 -1
  19. rucio/client/baseclient.py +33 -24
  20. rucio/client/client.py +45 -1
  21. rucio/client/didclient.py +5 -3
  22. rucio/client/downloadclient.py +6 -8
  23. rucio/client/replicaclient.py +0 -2
  24. rucio/client/richclient.py +317 -0
  25. rucio/client/rseclient.py +4 -4
  26. rucio/client/uploadclient.py +26 -12
  27. rucio/common/bittorrent.py +234 -0
  28. rucio/common/cache.py +66 -29
  29. rucio/common/checksum.py +168 -0
  30. rucio/common/client.py +122 -0
  31. rucio/common/config.py +22 -35
  32. rucio/common/constants.py +61 -3
  33. rucio/common/didtype.py +72 -24
  34. rucio/common/dumper/__init__.py +45 -38
  35. rucio/common/dumper/consistency.py +75 -30
  36. rucio/common/dumper/data_models.py +63 -19
  37. rucio/common/dumper/path_parsing.py +19 -8
  38. rucio/common/exception.py +65 -8
  39. rucio/common/extra.py +5 -10
  40. rucio/common/logging.py +13 -13
  41. rucio/common/pcache.py +8 -7
  42. rucio/common/plugins.py +59 -27
  43. rucio/common/policy.py +12 -3
  44. rucio/common/schema/__init__.py +84 -34
  45. rucio/common/schema/generic.py +0 -17
  46. rucio/common/schema/generic_multi_vo.py +0 -17
  47. rucio/common/test_rucio_server.py +12 -6
  48. rucio/common/types.py +132 -52
  49. rucio/common/utils.py +93 -643
  50. rucio/core/account_limit.py +14 -12
  51. rucio/core/authentication.py +2 -2
  52. rucio/core/config.py +23 -42
  53. rucio/core/credential.py +14 -15
  54. rucio/core/did.py +5 -1
  55. rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
  56. rucio/core/did_meta_plugins/filter_engine.py +62 -3
  57. rucio/core/did_meta_plugins/json_meta.py +2 -2
  58. rucio/core/did_meta_plugins/mongo_meta.py +43 -30
  59. rucio/core/did_meta_plugins/postgres_meta.py +75 -39
  60. rucio/core/identity.py +6 -5
  61. rucio/core/importer.py +4 -3
  62. rucio/core/lifetime_exception.py +2 -2
  63. rucio/core/lock.py +8 -7
  64. rucio/core/message.py +6 -0
  65. rucio/core/monitor.py +30 -29
  66. rucio/core/naming_convention.py +2 -2
  67. rucio/core/nongrid_trace.py +2 -2
  68. rucio/core/oidc.py +11 -9
  69. rucio/core/permission/__init__.py +79 -37
  70. rucio/core/permission/generic.py +1 -7
  71. rucio/core/permission/generic_multi_vo.py +1 -7
  72. rucio/core/quarantined_replica.py +4 -3
  73. rucio/core/replica.py +464 -139
  74. rucio/core/replica_sorter.py +55 -59
  75. rucio/core/request.py +34 -32
  76. rucio/core/rse.py +301 -97
  77. rucio/core/rse_counter.py +1 -2
  78. rucio/core/rse_expression_parser.py +7 -7
  79. rucio/core/rse_selector.py +9 -7
  80. rucio/core/rule.py +41 -40
  81. rucio/core/rule_grouping.py +42 -40
  82. rucio/core/scope.py +5 -4
  83. rucio/core/subscription.py +26 -28
  84. rucio/core/topology.py +11 -11
  85. rucio/core/trace.py +2 -2
  86. rucio/core/transfer.py +29 -15
  87. rucio/core/volatile_replica.py +4 -3
  88. rucio/daemons/atropos/atropos.py +1 -1
  89. rucio/daemons/auditor/__init__.py +2 -2
  90. rucio/daemons/auditor/srmdumps.py +6 -6
  91. rucio/daemons/automatix/automatix.py +32 -21
  92. rucio/daemons/badreplicas/necromancer.py +2 -2
  93. rucio/daemons/bb8/nuclei_background_rebalance.py +1 -1
  94. rucio/daemons/bb8/t2_background_rebalance.py +1 -1
  95. rucio/daemons/common.py +15 -25
  96. rucio/daemons/conveyor/finisher.py +2 -2
  97. rucio/daemons/conveyor/poller.py +18 -28
  98. rucio/daemons/conveyor/receiver.py +2 -2
  99. rucio/daemons/conveyor/stager.py +1 -0
  100. rucio/daemons/conveyor/submitter.py +3 -3
  101. rucio/daemons/hermes/hermes.py +91 -30
  102. rucio/daemons/judge/evaluator.py +2 -2
  103. rucio/daemons/oauthmanager/oauthmanager.py +3 -3
  104. rucio/daemons/reaper/dark_reaper.py +7 -3
  105. rucio/daemons/reaper/reaper.py +12 -16
  106. rucio/daemons/rsedecommissioner/config.py +1 -1
  107. rucio/daemons/rsedecommissioner/profiles/generic.py +5 -4
  108. rucio/daemons/rsedecommissioner/profiles/types.py +7 -6
  109. rucio/daemons/rsedecommissioner/rse_decommissioner.py +1 -1
  110. rucio/daemons/storage/consistency/actions.py +8 -6
  111. rucio/daemons/tracer/kronos.py +4 -4
  112. rucio/db/sqla/constants.py +5 -0
  113. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +4 -4
  114. rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
  115. rucio/db/sqla/models.py +157 -154
  116. rucio/db/sqla/session.py +58 -27
  117. rucio/db/sqla/types.py +2 -2
  118. rucio/db/sqla/util.py +2 -2
  119. rucio/gateway/account.py +18 -12
  120. rucio/gateway/account_limit.py +137 -60
  121. rucio/gateway/authentication.py +18 -12
  122. rucio/gateway/config.py +30 -20
  123. rucio/gateway/credential.py +9 -10
  124. rucio/gateway/did.py +70 -53
  125. rucio/gateway/dirac.py +6 -4
  126. rucio/gateway/exporter.py +3 -2
  127. rucio/gateway/heartbeat.py +6 -4
  128. rucio/gateway/identity.py +36 -51
  129. rucio/gateway/importer.py +3 -2
  130. rucio/gateway/lifetime_exception.py +3 -2
  131. rucio/gateway/meta_conventions.py +17 -6
  132. rucio/gateway/permission.py +4 -1
  133. rucio/gateway/quarantined_replica.py +3 -2
  134. rucio/gateway/replica.py +31 -22
  135. rucio/gateway/request.py +27 -18
  136. rucio/gateway/rse.py +69 -37
  137. rucio/gateway/rule.py +46 -26
  138. rucio/gateway/scope.py +3 -2
  139. rucio/gateway/subscription.py +14 -11
  140. rucio/gateway/vo.py +12 -8
  141. rucio/rse/__init__.py +3 -3
  142. rucio/rse/protocols/bittorrent.py +11 -1
  143. rucio/rse/protocols/cache.py +0 -11
  144. rucio/rse/protocols/dummy.py +0 -11
  145. rucio/rse/protocols/gfal.py +14 -9
  146. rucio/rse/protocols/globus.py +1 -1
  147. rucio/rse/protocols/http_cache.py +1 -1
  148. rucio/rse/protocols/posix.py +2 -2
  149. rucio/rse/protocols/protocol.py +84 -317
  150. rucio/rse/protocols/rclone.py +2 -1
  151. rucio/rse/protocols/rfio.py +10 -1
  152. rucio/rse/protocols/ssh.py +2 -1
  153. rucio/rse/protocols/storm.py +2 -13
  154. rucio/rse/protocols/webdav.py +74 -30
  155. rucio/rse/protocols/xrootd.py +2 -1
  156. rucio/rse/rsemanager.py +170 -53
  157. rucio/rse/translation.py +260 -0
  158. rucio/tests/common.py +23 -13
  159. rucio/tests/common_server.py +26 -9
  160. rucio/transfertool/bittorrent.py +15 -14
  161. rucio/transfertool/bittorrent_driver.py +5 -7
  162. rucio/transfertool/bittorrent_driver_qbittorrent.py +9 -8
  163. rucio/transfertool/fts3.py +20 -16
  164. rucio/transfertool/mock.py +2 -3
  165. rucio/vcsversion.py +4 -4
  166. rucio/version.py +7 -0
  167. rucio/web/rest/flaskapi/v1/accounts.py +17 -3
  168. rucio/web/rest/flaskapi/v1/auth.py +5 -5
  169. rucio/web/rest/flaskapi/v1/credentials.py +3 -2
  170. rucio/web/rest/flaskapi/v1/dids.py +21 -15
  171. rucio/web/rest/flaskapi/v1/identities.py +33 -9
  172. rucio/web/rest/flaskapi/v1/redirect.py +5 -4
  173. rucio/web/rest/flaskapi/v1/replicas.py +12 -8
  174. rucio/web/rest/flaskapi/v1/rses.py +15 -4
  175. rucio/web/rest/flaskapi/v1/traces.py +56 -19
  176. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/alembic.ini.template +1 -1
  177. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/alembic_offline.ini.template +1 -1
  178. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +3 -2
  179. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rucio.cfg.template +3 -19
  180. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +1 -18
  181. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/requirements.server.txt +97 -68
  182. rucio-37.0.0.data/scripts/rucio +133 -0
  183. rucio-37.0.0.data/scripts/rucio-admin +97 -0
  184. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-atropos +2 -2
  185. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-auditor +2 -1
  186. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-automatix +2 -2
  187. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-cache-client +17 -10
  188. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-receiver +1 -0
  189. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-kronos +1 -0
  190. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-minos +2 -2
  191. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-minos-temporary-expiration +2 -2
  192. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-necromancer +2 -2
  193. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-reaper +6 -6
  194. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-transmogrifier +2 -2
  195. rucio-37.0.0.dist-info/METADATA +92 -0
  196. {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/RECORD +237 -243
  197. {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/licenses/AUTHORS.rst +3 -0
  198. rucio/common/schema/atlas.py +0 -413
  199. rucio/common/schema/belleii.py +0 -408
  200. rucio/common/schema/domatpc.py +0 -401
  201. rucio/common/schema/escape.py +0 -426
  202. rucio/common/schema/icecube.py +0 -406
  203. rucio/core/permission/atlas.py +0 -1348
  204. rucio/core/permission/belleii.py +0 -1077
  205. rucio/core/permission/escape.py +0 -1078
  206. rucio/daemons/c3po/algorithms/__init__.py +0 -13
  207. rucio/daemons/c3po/algorithms/simple.py +0 -134
  208. rucio/daemons/c3po/algorithms/t2_free_space.py +0 -128
  209. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +0 -130
  210. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +0 -294
  211. rucio/daemons/c3po/c3po.py +0 -371
  212. rucio/daemons/c3po/collectors/agis.py +0 -108
  213. rucio/daemons/c3po/collectors/free_space.py +0 -81
  214. rucio/daemons/c3po/collectors/jedi_did.py +0 -57
  215. rucio/daemons/c3po/collectors/mock_did.py +0 -51
  216. rucio/daemons/c3po/collectors/network_metrics.py +0 -71
  217. rucio/daemons/c3po/collectors/workload.py +0 -112
  218. rucio/daemons/c3po/utils/__init__.py +0 -13
  219. rucio/daemons/c3po/utils/dataset_cache.py +0 -50
  220. rucio/daemons/c3po/utils/expiring_dataset_cache.py +0 -56
  221. rucio/daemons/c3po/utils/expiring_list.py +0 -62
  222. rucio/daemons/c3po/utils/popularity.py +0 -85
  223. rucio/daemons/c3po/utils/timeseries.py +0 -89
  224. rucio/rse/protocols/gsiftp.py +0 -92
  225. rucio-35.7.0.data/scripts/rucio-c3po +0 -85
  226. rucio-35.7.0.dist-info/METADATA +0 -72
  227. /rucio/{daemons/c3po → cli/bin_legacy}/__init__.py +0 -0
  228. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  229. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
  230. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  231. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  232. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  233. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  234. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  235. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  236. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  237. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/tools/bootstrap.py +0 -0
  238. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
  239. {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/tools/reset_database.py +0 -0
  240. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-abacus-account +0 -0
  241. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-abacus-collection-replica +0 -0
  242. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-abacus-rse +0 -0
  243. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-bb8 +0 -0
  244. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-cache-consumer +0 -0
  245. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-finisher +0 -0
  246. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-poller +0 -0
  247. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-preparer +0 -0
  248. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-stager +0 -0
  249. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-submitter +0 -0
  250. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-throttler +0 -0
  251. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-dark-reaper +0 -0
  252. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-dumper +0 -0
  253. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-follower +0 -0
  254. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-hermes +0 -0
  255. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-cleaner +0 -0
  256. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-evaluator +0 -0
  257. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-injector +0 -0
  258. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-repairer +0 -0
  259. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-oauth-manager +0 -0
  260. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-replica-recoverer +0 -0
  261. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-rse-decommissioner +0 -0
  262. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-storage-consistency-actions +0 -0
  263. {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-undertaker +0 -0
  264. {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/WHEEL +0 -0
  265. {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/licenses/LICENSE +0 -0
  266. {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/top_level.txt +0 -0
rucio/core/replica.py CHANGED
@@ -26,7 +26,7 @@ from json import dumps
26
26
  from re import match
27
27
  from struct import unpack
28
28
  from traceback import format_exc
29
- from typing import TYPE_CHECKING
29
+ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
30
30
 
31
31
  import requests
32
32
  from dogpile.cache.api import NO_VALUE
@@ -34,15 +34,15 @@ from sqlalchemy import and_, delete, exists, func, insert, not_, or_, union, upd
34
34
  from sqlalchemy.exc import DatabaseError, IntegrityError
35
35
  from sqlalchemy.orm import aliased
36
36
  from sqlalchemy.orm.exc import FlushError, NoResultFound
37
- from sqlalchemy.sql.expression import case, false, literal, literal_column, null, select, text, true
37
+ from sqlalchemy.sql.expression import ColumnElement, case, false, literal, literal_column, null, select, text, true
38
38
 
39
39
  import rucio.core.did
40
40
  import rucio.core.lock
41
41
  from rucio.common import exception
42
- from rucio.common.cache import make_region_memcached
42
+ from rucio.common.cache import MemcacheRegion
43
43
  from rucio.common.config import config_get, config_get_bool
44
44
  from rucio.common.constants import RseAttr, SuspiciousAvailability
45
- from rucio.common.types import InternalScope
45
+ from rucio.common.types import InternalAccount, InternalScope, LFNDict, is_str_list
46
46
  from rucio.common.utils import add_url_query, chunks, clean_pfns, str_to_date
47
47
  from rucio.core.credential import get_signed_url
48
48
  from rucio.core.message import add_messages
@@ -57,15 +57,16 @@ from rucio.db.sqla.util import temp_table_mngr
57
57
  from rucio.rse import rsemanager as rsemgr
58
58
 
59
59
  if TYPE_CHECKING:
60
- from collections.abc import Iterable, Iterator, Sequence
61
- from typing import Any, Optional
60
+ from collections.abc import Iterable, Iterator, Mapping, Sequence
62
61
 
62
+ from sqlalchemy.engine import Row
63
63
  from sqlalchemy.orm import Session
64
+ from sqlalchemy.sql.selectable import Select, Subquery
64
65
 
65
66
  from rucio.common.types import LoggerFunction
66
67
  from rucio.rse.protocols.protocol import RSEProtocol
67
68
 
68
- REGION = make_region_memcached(expiration_time=60)
69
+ REGION = MemcacheRegion(expiration_time=60)
69
70
  METRICS = MetricManager(module=__name__)
70
71
 
71
72
 
@@ -74,7 +75,14 @@ Association = namedtuple('Association', ['scope', 'name', 'child_scope', 'child_
74
75
 
75
76
 
76
77
  @read_session
77
- def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None, filter_=None, *, session: "Session"):
78
+ def get_bad_replicas_summary(
79
+ rse_expression: Optional[str] = None,
80
+ from_date: Optional[datetime] = None,
81
+ to_date: Optional[datetime] = None,
82
+ filter_: Optional[dict[str, Any]] = None,
83
+ *,
84
+ session: "Session"
85
+ ) -> list[dict[str, Any]]:
78
86
  """
79
87
  List the bad file replicas summary. Method used by the rucio-ui.
80
88
  :param rse_expression: The RSE expression.
@@ -94,11 +102,11 @@ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None,
94
102
  for rse in list_rses(filters=filter_, session=session):
95
103
  rse_clause.append(models.BadReplica.rse_id == rse['id'])
96
104
 
97
- if session.bind.dialect.name == 'oracle':
105
+ if session.bind.dialect.name == 'oracle': # type: ignore
98
106
  to_days = func.trunc(models.BadReplica.created_at, 'DD')
99
- elif session.bind.dialect.name == 'mysql':
107
+ elif session.bind.dialect.name == 'mysql': # type: ignore
100
108
  to_days = func.date(models.BadReplica.created_at)
101
- elif session.bind.dialect.name == 'postgresql':
109
+ elif session.bind.dialect.name == 'postgresql': # type: ignore
102
110
  to_days = func.date_trunc('day', models.BadReplica.created_at)
103
111
  else:
104
112
  to_days = func.strftime(models.BadReplica.created_at, '%Y-%m-%d')
@@ -137,7 +145,22 @@ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None,
137
145
 
138
146
 
139
147
  @read_session
140
- def __exist_replicas(rse_id, replicas, *, session: "Session"):
148
+ def __exist_replicas(
149
+ rse_id: str,
150
+ replicas: list[tuple[Optional[str], Optional[str], Optional[str]]],
151
+ *,
152
+ session: "Session"
153
+ ) -> list[
154
+ tuple
155
+ [
156
+ str,
157
+ str,
158
+ str,
159
+ bool,
160
+ bool,
161
+ Optional[int]
162
+ ]
163
+ ]:
141
164
  """
142
165
  Internal method to check if a replica exists at a given site.
143
166
  :param rse_id: The RSE id.
@@ -153,8 +176,8 @@ def __exist_replicas(rse_id, replicas, *, session: "Session"):
153
176
  """
154
177
 
155
178
  return_list = []
156
- path_clause = []
157
- did_clause = []
179
+ path_clause: list[ColumnElement[bool]] = []
180
+ did_clause: list[ColumnElement[bool]] = []
158
181
  for scope, name, path in replicas:
159
182
  if path:
160
183
  path_clause.append(models.RSEFileAssociation.path == path)
@@ -219,7 +242,17 @@ def __exist_replicas(rse_id, replicas, *, session: "Session"):
219
242
 
220
243
 
221
244
  @read_session
222
- def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than=None, older_than=None, limit=None, list_pfns=False, vo='def', *, session: "Session"):
245
+ def list_bad_replicas_status(
246
+ state: BadFilesStatus = BadFilesStatus.BAD,
247
+ rse_id: Optional[str] = None,
248
+ younger_than: Optional[datetime] = None,
249
+ older_than: Optional[datetime] = None,
250
+ limit: Optional[int] = None,
251
+ list_pfns: Optional[bool] = False,
252
+ vo: str = 'def',
253
+ *,
254
+ session: "Session"
255
+ ) -> list[dict[str, Any]]:
223
256
  """
224
257
  List the bad file replicas history states. Method used by the rucio-ui.
225
258
  :param state: The state of the file (SUSPICIOUS or BAD).
@@ -272,7 +305,18 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than
272
305
 
273
306
 
274
307
  @transactional_session
275
- def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesStatus.BAD, scheme='srm', force=False, logger: "LoggerFunction" = logging.log, *, session: "Session"):
308
+ def __declare_bad_file_replicas(
309
+ pfns: list[Union[str, dict[str, Any]]],
310
+ rse_id: str,
311
+ reason: str,
312
+ issuer: InternalAccount,
313
+ status: BadFilesStatus = BadFilesStatus.BAD,
314
+ scheme: str = 'srm',
315
+ force: bool = False,
316
+ logger: "LoggerFunction" = logging.log,
317
+ *,
318
+ session: "Session"
319
+ ) -> list[str]:
276
320
  """
277
321
  Declare a list of bad replicas.
278
322
 
@@ -285,11 +329,11 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
285
329
  :param force: boolean, if declaring BAD replica, ignore existing replica status in the bad_replicas table. Default: False
286
330
  :param session: The database session in use.
287
331
  """
288
- unknown_replicas = []
289
- replicas = []
290
- path_pfn_dict = {}
332
+ unknown_replicas: list[str] = []
333
+ replicas: list[dict[str, Any]] = []
334
+ path_pfn_dict: dict[str, str] = {}
291
335
 
292
- if len(pfns) > 0 and type(pfns[0]) is str:
336
+ if len(pfns) > 0 and is_str_list(pfns):
293
337
  # If pfns is a list of PFNs, the scope and names need to be extracted from the path
294
338
  rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
295
339
  proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
@@ -319,8 +363,8 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
319
363
  else:
320
364
  # If pfns is a list of replicas, just use scope, name and rse_id
321
365
  for pfn in pfns:
322
- replicas.append({'scope': pfn['scope'], 'name': pfn['name'], 'rse_id': rse_id, 'state': status})
323
- logger(logging.DEBUG, f"Declaring replica {pfn['scope']}:{pfn['name']} {status} at {rse_id} without path")
366
+ replicas.append({'scope': pfn['scope'], 'name': pfn['name'], 'rse_id': rse_id, 'state': status}) # type: ignore
367
+ logger(logging.DEBUG, f"Declaring replica {pfn['scope']}:{pfn['name']} {status} at {rse_id} without path") # type: ignore
324
368
 
325
369
  replicas_list = []
326
370
  for replica in replicas:
@@ -381,7 +425,15 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
381
425
 
382
426
 
383
427
  @transactional_session
384
- def add_bad_dids(dids, rse_id, reason, issuer, state=BadFilesStatus.BAD, *, session: "Session"):
428
+ def add_bad_dids(
429
+ dids: "Iterable[dict[str, Any]]",
430
+ rse_id: str,
431
+ reason: str,
432
+ issuer: InternalAccount,
433
+ state: BadFilesStatus = BadFilesStatus.BAD,
434
+ *,
435
+ session: "Session"
436
+ ) -> list[str]:
385
437
  """
386
438
  Declare a list of bad replicas.
387
439
 
@@ -438,8 +490,15 @@ def add_bad_dids(dids, rse_id, reason, issuer, state=BadFilesStatus.BAD, *, sess
438
490
 
439
491
 
440
492
  @transactional_session
441
- def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFilesStatus.BAD, force: bool = False, *,
442
- session: "Session"):
493
+ def declare_bad_file_replicas(
494
+ replicas: list[Union[str, dict[str, Any]]],
495
+ reason: str,
496
+ issuer: InternalAccount,
497
+ status: BadFilesStatus = BadFilesStatus.BAD,
498
+ force: bool = False,
499
+ *,
500
+ session: "Session"
501
+ ) -> dict[str, list[str]]:
443
502
  """
444
503
  Declare a list of bad replicas.
445
504
 
@@ -451,7 +510,7 @@ def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFil
451
510
  :param session: The database session in use.
452
511
  :returns: Dictionary {rse_id -> [replicas failed to declare with errors]}
453
512
  """
454
- unknown_replicas = {}
513
+ unknown_replicas: dict[str, list[str]] = {}
455
514
  if replicas:
456
515
  type_ = type(replicas[0])
457
516
  files_to_declare = {}
@@ -459,11 +518,11 @@ def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFil
459
518
  for replica in replicas:
460
519
  if not isinstance(replica, type_):
461
520
  raise exception.InvalidType('Replicas must be specified either as a list of string or a list of dicts')
462
- if type_ == str:
521
+ if type_ is str:
463
522
  scheme, files_to_declare, unknown_replicas = get_pfn_to_rse(replicas, vo=issuer.vo, session=session)
464
523
  else:
465
524
  for replica in replicas:
466
- rse_id = replica['rse_id']
525
+ rse_id = replica['rse_id'] # type: ignore
467
526
  files_to_declare.setdefault(rse_id, []).append(replica)
468
527
  for rse_id in files_to_declare:
469
528
  notdeclared = __declare_bad_file_replicas(files_to_declare[rse_id], rse_id, reason, issuer,
@@ -475,7 +534,12 @@ def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFil
475
534
 
476
535
 
477
536
  @read_session
478
- def get_pfn_to_rse(pfns, vo='def', *, session: "Session"):
537
+ def get_pfn_to_rse(
538
+ pfns: "Iterable[str]",
539
+ vo: str = 'def',
540
+ *,
541
+ session: "Session"
542
+ ) -> tuple[Optional[str], dict[str, Any], dict[str, list[str]]]:
479
543
  """
480
544
  Get the RSE associated to a list of PFNs.
481
545
 
@@ -552,7 +616,10 @@ def get_pfn_to_rse(pfns, vo='def', *, session: "Session"):
552
616
 
553
617
 
554
618
  @read_session
555
- def get_bad_replicas_backlog(*, session: "Session"):
619
+ def get_bad_replicas_backlog(
620
+ *,
621
+ session: "Session"
622
+ ) -> dict[str, int]:
556
623
  """
557
624
  Get the replica backlog by RSE.
558
625
 
@@ -587,7 +654,14 @@ def get_bad_replicas_backlog(*, session: "Session"):
587
654
 
588
655
 
589
656
  @read_session
590
- def list_bad_replicas(limit=10000, thread=None, total_threads=None, rses=None, *, session: "Session"):
657
+ def list_bad_replicas(
658
+ limit: int = 10000,
659
+ thread: Optional[int] = None,
660
+ total_threads: Optional[int] = None,
661
+ rses: Optional['Iterable[dict[str, Any]]'] = None,
662
+ *,
663
+ session: "Session"
664
+ ) -> list[dict[str, Any]]:
591
665
  """
592
666
  List RSE File replicas with no locks.
593
667
 
@@ -634,7 +708,13 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, rses=None, *
634
708
 
635
709
 
636
710
  @stream_session
637
- def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
711
+ def get_did_from_pfns(
712
+ pfns: "Iterable[str]",
713
+ rse_id: Optional[str] = None,
714
+ vo: str = 'def',
715
+ *,
716
+ session: "Session"
717
+ ) -> 'Iterator[dict[str, dict[str, Any]]]':
638
718
  """
639
719
  Get the DIDs associated to a PFN on one given RSE
640
720
 
@@ -656,7 +736,7 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
656
736
  pfns = dict_rse[rse_id]
657
737
  rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
658
738
  pfndict = {}
659
- proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
739
+ proto: RSEProtocol = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
660
740
  if rse_info['deterministic']:
661
741
  scope_proto = rsemgr.get_scope_protocol(vo=vo)
662
742
  parsed_pfn = proto.parse_pfns(pfns=pfns)
@@ -685,7 +765,10 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
685
765
  yield {pfndict[pfn]: {'scope': scope, 'name': name}}
686
766
 
687
767
 
688
- def _pick_n_random(nrandom, generator):
768
+ def _pick_n_random(
769
+ nrandom: int,
770
+ generator: 'Iterable[Any]'
771
+ ) -> 'Iterator[Any]':
689
772
  """
690
773
  Select n random elements from the generator
691
774
  """
@@ -720,7 +803,11 @@ def _pick_n_random(nrandom, generator):
720
803
  yield r
721
804
 
722
805
 
723
- def _list_files_wo_replicas(files_wo_replica, *, session: "Session"):
806
+ def _list_files_wo_replicas(
807
+ files_wo_replica: "Iterable[dict[str, Any]]",
808
+ *,
809
+ session: "Session"
810
+ ) -> 'Iterator[tuple[str, str, int, str, str]]':
724
811
  if files_wo_replica:
725
812
  file_wo_clause = []
726
813
  for file in sorted(files_wo_replica, key=lambda f: (f['scope'], f['name'])):
@@ -744,7 +831,7 @@ def _list_files_wo_replicas(files_wo_replica, *, session: "Session"):
744
831
  yield scope, name, bytes_, md5, adler32
745
832
 
746
833
 
747
- def get_vp_endpoint():
834
+ def get_vp_endpoint() -> str:
748
835
  """
749
836
  VP endpoint is the Virtual Placement server.
750
837
  Once VP is integrated in Rucio it won't be needed.
@@ -753,7 +840,11 @@ def get_vp_endpoint():
753
840
  return vp_endpoint
754
841
 
755
842
 
756
- def get_multi_cache_prefix(cache_site, filename, logger=logging.log):
843
+ def get_multi_cache_prefix(
844
+ cache_site: str,
845
+ filename: str,
846
+ logger: "LoggerFunction" = logging.log
847
+ ) -> str:
757
848
  """
758
849
  for a givent cache site and filename, return address of the cache node that
759
850
  should be prefixed.
@@ -780,10 +871,10 @@ def get_multi_cache_prefix(cache_site, filename, logger=logging.log):
780
871
  logger(logging.WARNING, 'In get_multi_cache_prefix, could not access {}. Excaption:{}'.format(vp_endpoint, re))
781
872
  return ''
782
873
 
783
- if cache_site not in x_caches:
874
+ if cache_site not in x_caches: # type: ignore
784
875
  return ''
785
876
 
786
- xcache_site = x_caches[cache_site]
877
+ xcache_site = x_caches[cache_site] # type: ignore
787
878
  h = float(
788
879
  unpack('Q', sha256(filename.encode('utf-8')).digest()[:8])[0]) / 2**64
789
880
  for irange in xcache_site['ranges']:
@@ -795,8 +886,8 @@ def get_multi_cache_prefix(cache_site, filename, logger=logging.log):
795
886
  def _get_list_replicas_protocols(
796
887
  rse_id: str,
797
888
  domain: str,
798
- schemes: "Sequence[str]",
799
- additional_schemes: "Sequence[str]",
889
+ schemes: Optional[list[str]],
890
+ additional_schemes: "Iterable[str]",
800
891
  session: "Session"
801
892
  ) -> "list[tuple[str, RSEProtocol, int]]":
802
893
  """
@@ -851,9 +942,9 @@ def _build_list_replicas_pfn(
851
942
  protocol: "RSEProtocol",
852
943
  path: str,
853
944
  sign_urls: bool,
854
- signature_lifetime: int,
855
- client_location: "dict[str, Any]",
856
- logger=logging.log,
945
+ signature_lifetime: Optional[int],
946
+ client_location: Optional[dict[str, Any]],
947
+ logger: "LoggerFunction" = logging.log,
857
948
  *,
858
949
  session: "Session",
859
950
  ) -> str:
@@ -862,9 +953,12 @@ def _build_list_replicas_pfn(
862
953
  If needed, sign the PFN url
863
954
  If relevant, add the server-side root proxy to the pfn url
864
955
  """
865
- pfn: str = list(protocol.lfns2pfns(lfns={'scope': scope.external,
866
- 'name': name,
867
- 'path': path}).values())[0]
956
+ lfn: LFNDict = {
957
+ 'scope': scope.external, # type: ignore (scope.external might be None)
958
+ 'name': name,
959
+ 'path': path
960
+ }
961
+ pfn: str = list(protocol.lfns2pfns(lfns=lfn).values())[0]
868
962
 
869
963
  # do we need to sign the URLs?
870
964
  if sign_urls and protocol.attributes['scheme'] == 'https':
@@ -908,7 +1002,7 @@ def _build_list_replicas_pfn(
908
1002
  # don't forget to mangle gfal-style davs URL into generic https URL
909
1003
  pfn = f"root://{root_proxy_internal}//{pfn.replace('davs://', 'https://')}"
910
1004
 
911
- simulate_multirange = get_rse_attribute(rse_id, RseAttr.SIMULATE_MULTIRANGE)
1005
+ simulate_multirange = get_rse_attribute(rse_id, RseAttr.SIMULATE_MULTIRANGE, session=session)
912
1006
 
913
1007
  if simulate_multirange is not None:
914
1008
  try:
@@ -925,8 +1019,21 @@ def _build_list_replicas_pfn(
925
1019
  return pfn
926
1020
 
927
1021
 
928
- def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_location, domain,
929
- sign_urls, signature_lifetime, resolve_parents, filters, by_rse_name, *, session: "Session"):
1022
+ def _list_replicas(
1023
+ replicas: "Iterable[tuple]",
1024
+ show_pfns: bool,
1025
+ schemes: Optional[list[str]],
1026
+ files_wo_replica: "Iterable[dict[str, Any]]",
1027
+ client_location: Optional[dict[str, Any]],
1028
+ domain: Optional[str],
1029
+ sign_urls: bool,
1030
+ signature_lifetime: Optional[int],
1031
+ resolve_parents: bool,
1032
+ filters: dict[str, Any],
1033
+ by_rse_name: bool,
1034
+ *,
1035
+ session: "Session"
1036
+ ) -> "Iterator[dict[str, Any]]":
930
1037
 
931
1038
  # the `domain` variable name will be re-used throughout the function with different values
932
1039
  input_domain = domain
@@ -948,7 +1055,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
948
1055
  pfns = {}
949
1056
  for scope, name, archive_scope, archive_name, bytes_, md5, adler32, path, state, rse_id, rse, rse_type, volatile in replica_group:
950
1057
  if isinstance(archive_scope, str):
951
- archive_scope = InternalScope(archive_scope, fromExternal=False)
1058
+ archive_scope = InternalScope(archive_scope, from_external=False)
952
1059
 
953
1060
  is_archive = bool(archive_scope and archive_name)
954
1061
 
@@ -1005,7 +1112,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
1005
1112
  try:
1006
1113
  path = pfns_cache['%s:%s:%s' % (protocol.attributes['determinism_type'], t_scope.internal, t_name)]
1007
1114
  except KeyError: # No cache entry scope:name found for this protocol
1008
- path = protocol._get_path(t_scope, t_name)
1115
+ path = protocol._get_path(t_scope, t_name) # type: ignore (t_scope is InternalScope instead of str)
1009
1116
  pfns_cache['%s:%s:%s' % (protocol.attributes['determinism_type'], t_scope.internal, t_name)] = path
1010
1117
 
1011
1118
  try:
@@ -1086,24 +1193,24 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
1086
1193
  @stream_session
1087
1194
  def list_replicas(
1088
1195
  dids: "Sequence[dict[str, Any]]",
1089
- schemes: "Optional[list[str]]" = None,
1196
+ schemes: Optional[list[str]] = None,
1090
1197
  unavailable: bool = False,
1091
- request_id: "Optional[str]" = None,
1198
+ request_id: Optional[str] = None,
1092
1199
  ignore_availability: bool = True,
1093
1200
  all_states: bool = False,
1094
1201
  pfns: bool = True,
1095
- rse_expression: "Optional[str]" = None,
1096
- client_location: "Optional[dict[str, Any]]" = None,
1097
- domain: "Optional[str]" = None,
1202
+ rse_expression: Optional[str] = None,
1203
+ client_location: Optional[dict[str, Any]] = None,
1204
+ domain: Optional[str] = None,
1098
1205
  sign_urls: bool = False,
1099
1206
  signature_lifetime: "Optional[int]" = None,
1100
1207
  resolve_archives: bool = True,
1101
1208
  resolve_parents: bool = False,
1102
- nrandom: "Optional[int]" = None,
1103
- updated_after: "Optional[datetime]" = None,
1209
+ nrandom: Optional[int] = None,
1210
+ updated_after: Optional[datetime] = None,
1104
1211
  by_rse_name: bool = False,
1105
1212
  *, session: "Session",
1106
- ):
1213
+ ) -> 'Iterator':
1107
1214
  """
1108
1215
  List file replicas for a list of data identifiers (DIDs).
1109
1216
 
@@ -1185,7 +1292,11 @@ def list_replicas(
1185
1292
 
1186
1293
  return stmt.subquery()
1187
1294
 
1188
- def _resolve_collection_files(temp_table, *, session: "Session"):
1295
+ def _resolve_collection_files(
1296
+ temp_table: Any,
1297
+ *,
1298
+ session: "Session"
1299
+ ) -> tuple[int, Any]:
1189
1300
  """
1190
1301
  Find all FILE dids contained in collections from temp_table and return them in a newly
1191
1302
  created temporary table.
@@ -1202,7 +1313,10 @@ def list_replicas(
1202
1313
 
1203
1314
  return session.execute(stmt).rowcount, resolved_files_temp_table
1204
1315
 
1205
- def _list_replicas_for_collection_files_stmt(temp_table, replicas_subquery):
1316
+ def _list_replicas_for_collection_files_stmt(
1317
+ temp_table: Any,
1318
+ replicas_subquery: "Subquery"
1319
+ ) -> "Select":
1206
1320
  """
1207
1321
  Build a query for listing replicas of files resolved from containers/datasets
1208
1322
 
@@ -1229,7 +1343,10 @@ def list_replicas(
1229
1343
  replicas_subquery.c.name == temp_table.name),
1230
1344
  )
1231
1345
 
1232
- def _list_replicas_for_constituents_stmt(temp_table, replicas_subquery):
1346
+ def _list_replicas_for_constituents_stmt(
1347
+ temp_table: Any,
1348
+ replicas_subquery: "Subquery"
1349
+ ) -> "Select":
1233
1350
  """
1234
1351
  Build a query for listing replicas of archives containing the files(constituents) given as input.
1235
1352
  i.e. for a file scope:file.log which exists in scope:archive.tar.gz, it will return the replicas
@@ -1266,7 +1383,10 @@ def list_replicas(
1266
1383
  replicas_subquery.c.name == models.ConstituentAssociation.name),
1267
1384
  )
1268
1385
 
1269
- def _list_replicas_for_input_files_stmt(temp_table, replicas_subquery):
1386
+ def _list_replicas_for_input_files_stmt(
1387
+ temp_table: Any,
1388
+ replicas_subquery: "Subquery"
1389
+ ) -> "Select":
1270
1390
  """
1271
1391
  Builds a query which list the replicas of FILEs from users input, but ignores
1272
1392
  collections in the same input.
@@ -1300,7 +1420,11 @@ def list_replicas(
1300
1420
  replicas_subquery.c.name == temp_table.name),
1301
1421
  )
1302
1422
 
1303
- def _inspect_dids(temp_table, *, session: "Session"):
1423
+ def _inspect_dids(
1424
+ temp_table: Any,
1425
+ *,
1426
+ session: "Session"
1427
+ ) -> tuple[int, int, int]:
1304
1428
  """
1305
1429
  Find how many files, collections and constituents are among the dids in the temp_table
1306
1430
  """
@@ -1328,7 +1452,7 @@ def list_replicas(
1328
1452
  else:
1329
1453
  filter_ = {'vo': 'def'}
1330
1454
 
1331
- dids = {(did['scope'], did['name']): did for did in dids} # Deduplicate input
1455
+ dids = {(did['scope'], did['name']): did for did in dids} # type: ignore (Deduplicate input)
1332
1456
  if not dids:
1333
1457
  return
1334
1458
 
@@ -1382,17 +1506,17 @@ def list_replicas(
1382
1506
  # (for example: scheme; or client_location/domain). We don't have any guarantee that
1383
1507
  # those, python, filters will not drop the replicas which we just selected randomly.
1384
1508
  stmt = select(
1385
- resolved_files_temp_table.scope.label('scope'),
1386
- resolved_files_temp_table.name.label('name'),
1509
+ resolved_files_temp_table.scope.label('scope'), # type: ignore (resolved_files_temp_table might be None)
1510
+ resolved_files_temp_table.name.label('name'), # type: ignore (resolved_files_temp_table might be None)
1387
1511
  ).where(
1388
1512
  exists(
1389
1513
  select(1)
1390
1514
  ).where(
1391
- replicas_subquery.c.scope == resolved_files_temp_table.scope,
1392
- replicas_subquery.c.name == resolved_files_temp_table.name
1515
+ replicas_subquery.c.scope == resolved_files_temp_table.scope, # type: ignore (resolved_files_temp_table might be None)
1516
+ replicas_subquery.c.name == resolved_files_temp_table.name # type: ignore (resolved_files_temp_table might be None)
1393
1517
  )
1394
1518
  ).order_by(
1395
- literal_column('dbms_random.value') if session.bind.dialect.name == 'oracle' else func.random()
1519
+ literal_column('dbms_random.value') if session.bind.dialect.name == 'oracle' else func.random() # type: ignore
1396
1520
  ).limit(
1397
1521
  # slightly overshoot to reduce the probability that python-side filtering will
1398
1522
  # leave us with less than nrandom replicas.
@@ -1416,7 +1540,7 @@ def list_replicas(
1416
1540
  random_replicas = list(
1417
1541
  _pick_n_random(
1418
1542
  nrandom,
1419
- _list_replicas(replica_tuples, pfns, schemes, [], client_location, domain,
1543
+ _list_replicas(replica_tuples, pfns, schemes, [], client_location, domain, # type: ignore (replica_tuples, pending SQLA2.1: https://github.com/rucio/rucio/discussions/6615)
1420
1544
  sign_urls, signature_lifetime, resolve_parents, filter_, by_rse_name, session=session)
1421
1545
  )
1422
1546
  )
@@ -1432,7 +1556,7 @@ def list_replicas(
1432
1556
  stmt = replica_sources[0].order_by('scope', 'name')
1433
1557
  replica_tuples = session.execute(stmt)
1434
1558
  else:
1435
- if session.bind.dialect.name == 'mysql':
1559
+ if session.bind.dialect.name == 'mysql': # type: ignore
1436
1560
  # On mysql, perform both queries independently and merge their result in python.
1437
1561
  # The union query fails with "Can't reopen table"
1438
1562
  replica_tuples = heapq.merge(
@@ -1444,14 +1568,20 @@ def list_replicas(
1444
1568
  replica_tuples = session.execute(stmt)
1445
1569
 
1446
1570
  yield from _pick_n_random(
1447
- nrandom,
1448
- _list_replicas(replica_tuples, pfns, schemes, [], client_location, domain,
1571
+ nrandom, # type: ignore (nrandom is not None)
1572
+ _list_replicas(replica_tuples, pfns, schemes, [], client_location, domain, # type: ignore (replica_tuples, pending SQLA2.1: https://github.com/rucio/rucio/discussions/6615)
1449
1573
  sign_urls, signature_lifetime, resolve_parents, filter_, by_rse_name, session=session)
1450
1574
  )
1451
1575
 
1452
1576
 
1453
1577
  @transactional_session
1454
- def __bulk_add_new_file_dids(files, account, dataset_meta=None, *, session: "Session"):
1578
+ def __bulk_add_new_file_dids(
1579
+ files: "Iterable[dict[str, Any]]",
1580
+ account: InternalAccount,
1581
+ dataset_meta: Optional["Mapping[str, Any]"] = None,
1582
+ *,
1583
+ session: "Session"
1584
+ ) -> Literal[True]:
1455
1585
  """
1456
1586
  Bulk add new dids.
1457
1587
 
@@ -1498,7 +1628,13 @@ def __bulk_add_new_file_dids(files, account, dataset_meta=None, *, session: "Ses
1498
1628
 
1499
1629
 
1500
1630
  @transactional_session
1501
- def __bulk_add_file_dids(files, account, dataset_meta=None, *, session: "Session"):
1631
+ def __bulk_add_file_dids(
1632
+ files: "Iterable[dict[str, Any]]",
1633
+ account: InternalAccount,
1634
+ dataset_meta: Optional["Mapping[str, Any]"] = None,
1635
+ *,
1636
+ session: "Session"
1637
+ ) -> list[dict[str, Any]]:
1502
1638
  """
1503
1639
  Bulk add new dids.
1504
1640
 
@@ -1542,12 +1678,12 @@ def __bulk_add_file_dids(files, account, dataset_meta=None, *, session: "Session
1542
1678
  return new_files + available_files
1543
1679
 
1544
1680
 
1545
- def tombstone_from_delay(tombstone_delay):
1681
+ def tombstone_from_delay(tombstone_delay: Optional[Union[str, timedelta]]) -> Optional[datetime]:
1546
1682
  # Tolerate None for tombstone_delay
1547
1683
  if not tombstone_delay:
1548
1684
  return None
1549
1685
 
1550
- tombstone_delay = timedelta(seconds=int(tombstone_delay))
1686
+ tombstone_delay = timedelta(seconds=int(tombstone_delay)) # type: ignore
1551
1687
 
1552
1688
  if not tombstone_delay:
1553
1689
  return None
@@ -1559,7 +1695,13 @@ def tombstone_from_delay(tombstone_delay):
1559
1695
 
1560
1696
 
1561
1697
  @transactional_session
1562
- def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
1698
+ def __bulk_add_replicas(
1699
+ rse_id: str,
1700
+ files: "Iterable[dict[str, Any]]",
1701
+ account: InternalAccount,
1702
+ *,
1703
+ session: "Session"
1704
+ ) -> tuple[int, int]:
1563
1705
  """
1564
1706
  Bulk add new dids.
1565
1707
 
@@ -1630,8 +1772,15 @@ def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
1630
1772
 
1631
1773
 
1632
1774
  @transactional_session
1633
- def add_replicas(rse_id, files, account, ignore_availability=True,
1634
- dataset_meta=None, *, session: "Session"):
1775
+ def add_replicas(
1776
+ rse_id: str,
1777
+ files: "Iterable[dict[str, Any]]",
1778
+ account: InternalAccount,
1779
+ ignore_availability: bool = True,
1780
+ dataset_meta: Optional["Mapping[str, Any]"] = None,
1781
+ *,
1782
+ session: "Session"
1783
+ ) -> None:
1635
1784
  """
1636
1785
  Bulk add file replicas.
1637
1786
 
@@ -1640,8 +1789,6 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
1640
1789
  :param account: The account owner.
1641
1790
  :param ignore_availability: Ignore the RSE blocklisting.
1642
1791
  :param session: The database session in use.
1643
-
1644
- :returns: list of replicas.
1645
1792
  """
1646
1793
 
1647
1794
  def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=None):
@@ -1685,18 +1832,24 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
1685
1832
  else:
1686
1833
  # Check that the pfns match to the expected pfns
1687
1834
  lfns = [{'scope': i['scope'].external, 'name': i['name']} for i in files if i['pfn'].startswith(scheme)]
1688
- pfns[scheme] = clean_pfns(pfns[scheme])
1835
+ pfns[scheme] = set(clean_pfns(pfns[scheme]))
1836
+ expected_pfns = set()
1689
1837
 
1690
1838
  for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan'):
1691
- pfns[scheme] = list(set(pfns[scheme]) - set(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=protocol_attr)))
1839
+ expected_pfns.update(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan',
1840
+ protocol_attr=protocol_attr))
1841
+ pfns[scheme] -= expected_pfns
1692
1842
 
1693
1843
  if len(pfns[scheme]) > 0:
1694
1844
  for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='lan'):
1695
- pfns[scheme] = list(set(pfns[scheme]) - set(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan', protocol_attr=protocol_attr)))
1845
+ expected_pfns.update(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan',
1846
+ protocol_attr=protocol_attr))
1847
+ pfns[scheme] -= expected_pfns
1696
1848
 
1697
1849
  if len(pfns[scheme]) > 0:
1698
1850
  # pfns not found in wan or lan
1699
- raise exception.InvalidPath('One of the PFNs provided does not match the Rucio expected PFN : %s (%s)' % (str(pfns[scheme]), str(lfns)))
1851
+ pfns_scheme = pfns[scheme]
1852
+ raise exception.InvalidPath(f"One of the PFNs provided {pfns_scheme!r} for {lfns!r} does not match the Rucio expected PFNs: {expected_pfns!r}")
1700
1853
 
1701
1854
  nbfiles, bytes_ = __bulk_add_replicas(rse_id=rse_id, files=files, account=account, session=session)
1702
1855
  increase(rse_id=rse_id, files=nbfiles, bytes_=bytes_, session=session)
@@ -1709,16 +1862,16 @@ def add_replica(
1709
1862
  name: str,
1710
1863
  bytes_: int,
1711
1864
  account: models.InternalAccount,
1712
- adler32: "Optional[str]" = None,
1713
- md5: "Optional[str]" = None,
1714
- dsn: "Optional[str]" = None,
1715
- pfn: "Optional[str]" = None,
1716
- meta: "Optional[dict[str, Any]]" = None,
1717
- rules: "Optional[list[dict[str, Any]]]" = None,
1865
+ adler32: Optional[str] = None,
1866
+ md5: Optional[str] = None,
1867
+ dsn: Optional[str] = None,
1868
+ pfn: Optional[str] = None,
1869
+ meta: Optional[dict[str, Any]] = None,
1870
+ rules: Optional[list[dict[str, Any]]] = None,
1718
1871
  tombstone: "Optional[datetime]" = None,
1719
1872
  *,
1720
1873
  session: "Session"
1721
- ) -> "list[dict[str, Any]]":
1874
+ ) -> list[dict[str, Any]]:
1722
1875
  """
1723
1876
  Add File replica.
1724
1877
 
@@ -1748,7 +1901,13 @@ def add_replica(
1748
1901
 
1749
1902
  @METRICS.time_it
1750
1903
  @transactional_session
1751
- def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Session"):
1904
+ def delete_replicas(
1905
+ rse_id: str,
1906
+ files: Optional["Sequence[dict[str, Any]]"],
1907
+ ignore_availability: bool = True,
1908
+ *,
1909
+ session: "Session"
1910
+ ) -> None:
1752
1911
  """
1753
1912
  Delete file replicas.
1754
1913
 
@@ -1847,7 +2006,15 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
1847
2006
 
1848
2007
 
1849
2008
  @transactional_session
1850
- def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_table2, association_temp_table, rse_id, files, *, session: "Session"):
2009
+ def __cleanup_after_replica_deletion(
2010
+ scope_name_temp_table: Any,
2011
+ scope_name_temp_table2: Any,
2012
+ association_temp_table: Any,
2013
+ rse_id: str,
2014
+ files: "Iterable[dict[str, Any]]",
2015
+ *,
2016
+ session: "Session"
2017
+ ) -> None:
1851
2018
  """
1852
2019
  Perform update of collections/archive associations/dids after the removal of their replicas
1853
2020
  :param rse_id: the rse id
@@ -2336,7 +2503,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
2336
2503
 
2337
2504
 
2338
2505
  @transactional_session
2339
- def get_replica(rse_id, scope, name, *, session: "Session"):
2506
+ def get_replica(
2507
+ rse_id: str,
2508
+ scope: InternalScope,
2509
+ name: str,
2510
+ *,
2511
+ session: "Session"
2512
+ ) -> dict[str, Any]:
2340
2513
  """
2341
2514
  Get File replica.
2342
2515
 
@@ -2361,7 +2534,15 @@ def get_replica(rse_id, scope, name, *, session: "Session"):
2361
2534
 
2362
2535
 
2363
2536
  @transactional_session
2364
- def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_seconds=600, only_delete_obsolete=False, *, session: "Session"):
2537
+ def list_and_mark_unlocked_replicas(
2538
+ limit: int,
2539
+ bytes_: Optional[int] = None,
2540
+ rse_id: Optional[str] = None,
2541
+ delay_seconds: int = 600,
2542
+ only_delete_obsolete: bool = False,
2543
+ *,
2544
+ session: "Session"
2545
+ ) -> list[dict[str, Any]]:
2365
2546
  """
2366
2547
  List RSE File replicas with no locks.
2367
2548
 
@@ -2478,7 +2659,7 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2478
2659
  if len(rows) >= limit or (not only_delete_obsolete and needed_space is not None and total_bytes > needed_space):
2479
2660
  break
2480
2661
  if state != ReplicaState.UNAVAILABLE:
2481
- total_bytes += bytes_
2662
+ total_bytes += bytes_ # type: ignore
2482
2663
 
2483
2664
  rows.append({'scope': scope, 'name': name, 'path': path,
2484
2665
  'bytes': bytes_, 'tombstone': tombstone,
@@ -2513,7 +2694,12 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2513
2694
 
2514
2695
 
2515
2696
  @transactional_session
2516
- def update_replicas_states(replicas, nowait=False, *, session: "Session"):
2697
+ def update_replicas_states(
2698
+ replicas: "Iterable[dict[str, Any]]",
2699
+ nowait: bool = False,
2700
+ *,
2701
+ session: "Session"
2702
+ ) -> bool:
2517
2703
  """
2518
2704
  Update File replica information and state.
2519
2705
 
@@ -2614,7 +2800,11 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
2614
2800
 
2615
2801
 
2616
2802
  @transactional_session
2617
- def touch_replica(replica, *, session: "Session"):
2803
+ def touch_replica(
2804
+ replica: dict[str, Any],
2805
+ *,
2806
+ session: "Session"
2807
+ ) -> bool:
2618
2808
  """
2619
2809
  Update the accessed_at timestamp of the given file replica/did but don't wait if row is locked.
2620
2810
 
@@ -2699,7 +2889,14 @@ def touch_replica(replica, *, session: "Session"):
2699
2889
 
2700
2890
 
2701
2891
  @transactional_session
2702
- def update_replica_state(rse_id, scope, name, state, *, session: "Session"):
2892
+ def update_replica_state(
2893
+ rse_id: str,
2894
+ scope: InternalScope,
2895
+ name: str,
2896
+ state: BadFilesStatus,
2897
+ *,
2898
+ session: "Session"
2899
+ ) -> bool:
2703
2900
  """
2704
2901
  Update File replica information and state.
2705
2902
 
@@ -2713,7 +2910,14 @@ def update_replica_state(rse_id, scope, name, state, *, session: "Session"):
2713
2910
 
2714
2911
 
2715
2912
  @transactional_session
2716
- def get_and_lock_file_replicas(scope, name, nowait=False, restrict_rses=None, *, session: "Session"):
2913
+ def get_and_lock_file_replicas(
2914
+ scope: InternalScope,
2915
+ name: str,
2916
+ nowait: bool = False,
2917
+ restrict_rses: Optional["Sequence[str]"] = None,
2918
+ *,
2919
+ session: "Session"
2920
+ ) -> "Sequence[models.RSEFileAssociation]":
2717
2921
  """
2718
2922
  Get file replicas for a specific scope:name.
2719
2923
 
@@ -2743,7 +2947,13 @@ def get_and_lock_file_replicas(scope, name, nowait=False, restrict_rses=None, *,
2743
2947
 
2744
2948
 
2745
2949
  @transactional_session
2746
- def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
2950
+ def get_source_replicas(
2951
+ scope: InternalScope,
2952
+ name: str,
2953
+ source_rses: Optional["Sequence[str]"] = None,
2954
+ *,
2955
+ session: "Session"
2956
+ ) -> "Sequence[str]":
2747
2957
  """
2748
2958
  Get source replicas for a specific scope:name.
2749
2959
 
@@ -2772,9 +2982,16 @@ def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
2772
2982
 
2773
2983
 
2774
2984
  @transactional_session
2775
- def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_rses=None,
2776
- total_threads=None, thread_id=None,
2777
- *, session: "Session"):
2985
+ def get_and_lock_file_replicas_for_dataset(
2986
+ scope: InternalScope,
2987
+ name: str,
2988
+ nowait: bool = False,
2989
+ restrict_rses: Optional["Sequence[str]"] = None,
2990
+ total_threads: Optional[int] = None,
2991
+ thread_id: Optional[int] = None,
2992
+ *,
2993
+ session: "Session"
2994
+ ) -> tuple[list[dict[str, Any]], dict[tuple[InternalScope, str], Any]]:
2778
2995
  """
2779
2996
  Get file replicas for all files of a dataset.
2780
2997
 
@@ -2812,7 +3029,7 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
2812
3029
  if restrict_rses is not None and len(restrict_rses) < 10:
2813
3030
  rse_clause = [models.RSEFileAssociation.rse_id == rse_id for rse_id in restrict_rses]
2814
3031
 
2815
- if session.bind.dialect.name == 'postgresql':
3032
+ if session.bind.dialect.name == 'postgresql': # type: ignore
2816
3033
  if total_threads and total_threads > 1:
2817
3034
  base_stmt = filter_thread_work(session=session,
2818
3035
  query=base_stmt,
@@ -2876,9 +3093,15 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
2876
3093
 
2877
3094
 
2878
3095
  @transactional_session
2879
- def get_source_replicas_for_dataset(scope, name, source_rses=None,
2880
- total_threads=None, thread_id=None,
2881
- *, session: "Session"):
3096
+ def get_source_replicas_for_dataset(
3097
+ scope: InternalScope,
3098
+ name: str,
3099
+ source_rses: Optional["Sequence[str]"] = None,
3100
+ total_threads: Optional[int] = None,
3101
+ thread_id: Optional[int] = None,
3102
+ *,
3103
+ session: "Session"
3104
+ ) -> dict[tuple[InternalScope, str], Any]:
2882
3105
  """
2883
3106
  Get file replicas for all files of a dataset.
2884
3107
 
@@ -2955,7 +3178,11 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None,
2955
3178
 
2956
3179
 
2957
3180
  @read_session
2958
- def get_replica_atime(replica, *, session: "Session"):
3181
+ def get_replica_atime(
3182
+ replica: dict[str, Any],
3183
+ *,
3184
+ session: "Session"
3185
+ ) -> Optional[datetime]:
2959
3186
  """
2960
3187
  Get the accessed_at timestamp for a replica. Just for testing.
2961
3188
  :param replicas: List of dictionaries {scope, name, rse_id, path}
@@ -2978,7 +3205,11 @@ def get_replica_atime(replica, *, session: "Session"):
2978
3205
 
2979
3206
 
2980
3207
  @transactional_session
2981
- def touch_collection_replicas(collection_replicas, *, session: "Session"):
3208
+ def touch_collection_replicas(
3209
+ collection_replicas: "Iterable[dict[str, Any]]",
3210
+ *,
3211
+ session: "Session"
3212
+ ) -> bool:
2982
3213
  """
2983
3214
  Update the accessed_at timestamp of the given collection replicas.
2984
3215
 
@@ -3010,7 +3241,13 @@ def touch_collection_replicas(collection_replicas, *, session: "Session"):
3010
3241
 
3011
3242
 
3012
3243
  @stream_session
3013
- def list_dataset_replicas(scope, name, deep=False, *, session: "Session"):
3244
+ def list_dataset_replicas(
3245
+ scope: "InternalScope",
3246
+ name: str,
3247
+ deep: bool = False,
3248
+ *,
3249
+ session: "Session"
3250
+ ) -> "Iterator[dict[str, Any]]":
3014
3251
  """
3015
3252
  :param scope: The scope of the dataset.
3016
3253
  :param name: The name of the dataset.
@@ -3184,7 +3421,11 @@ def list_dataset_replicas(scope, name, deep=False, *, session: "Session"):
3184
3421
 
3185
3422
 
3186
3423
  @stream_session
3187
- def list_dataset_replicas_bulk(names_by_intscope, *, session: "Session"):
3424
+ def list_dataset_replicas_bulk(
3425
+ names_by_intscope: dict[str, Any],
3426
+ *,
3427
+ session: "Session"
3428
+ ) -> "Iterator[dict[str, Any]]":
3188
3429
  """
3189
3430
  :param names_by_intscope: The dictionary of internal scopes pointing at the list of names.
3190
3431
  :param session: Database session to use.
@@ -3228,7 +3469,14 @@ def list_dataset_replicas_bulk(names_by_intscope, *, session: "Session"):
3228
3469
 
3229
3470
 
3230
3471
  @stream_session
3231
- def list_dataset_replicas_vp(scope, name, deep=False, *, session: "Session", logger=logging.log):
3472
+ def list_dataset_replicas_vp(
3473
+ scope: InternalScope,
3474
+ name: str,
3475
+ deep: bool = False,
3476
+ *,
3477
+ session: "Session",
3478
+ logger: "LoggerFunction" = logging.log
3479
+ ) -> Union[list[str], "Iterator[dict[str, Any]]"]:
3232
3480
  """
3233
3481
  List dataset replicas for a DID (scope:name) using the
3234
3482
  Virtual Placement service.
@@ -3284,7 +3532,13 @@ def list_dataset_replicas_vp(scope, name, deep=False, *, session: "Session", log
3284
3532
 
3285
3533
 
3286
3534
  @stream_session
3287
- def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session"):
3535
+ def list_datasets_per_rse(
3536
+ rse_id: str,
3537
+ filters: Optional[dict[str, Any]] = None,
3538
+ limit: Optional[int] = None,
3539
+ *,
3540
+ session: "Session"
3541
+ ) -> "Iterator[dict[str, Any]]":
3288
3542
  """
3289
3543
  List datasets at a RSE.
3290
3544
 
@@ -3317,9 +3571,9 @@ def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session
3317
3571
 
3318
3572
  for (k, v) in filters and filters.items() or []:
3319
3573
  if k == 'name' or k == 'scope':
3320
- v_str = v if k != 'scope' else v.internal
3574
+ v_str = v if k != 'scope' else v.internal # type: ignore
3321
3575
  if '*' in v_str or '%' in v_str:
3322
- if session.bind.dialect.name == 'postgresql': # PostgreSQL escapes automatically
3576
+ if session.bind.dialect.name == 'postgresql': # type: ignore | PostgreSQL escapes automatically
3323
3577
  stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%')))
3324
3578
  else:
3325
3579
  stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%'), escape='\\'))
@@ -3345,7 +3599,7 @@ def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session
3345
3599
  @stream_session
3346
3600
  def list_replicas_per_rse(
3347
3601
  rse_id: str,
3348
- limit: "Optional[int]" = None,
3602
+ limit: Optional[int] = None,
3349
3603
  *,
3350
3604
  session: "Session"
3351
3605
  ) -> "Iterator[dict[str, Any]]":
@@ -3364,7 +3618,13 @@ def list_replicas_per_rse(
3364
3618
 
3365
3619
 
3366
3620
  @transactional_session
3367
- def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=None, *, session: "Session"):
3621
+ def get_cleaned_updated_collection_replicas(
3622
+ total_workers: int,
3623
+ worker_number: int,
3624
+ limit: Optional[int] = None,
3625
+ *,
3626
+ session: "Session"
3627
+ ) -> list[dict[str, Any]]:
3368
3628
  """
3369
3629
  Get update request for collection replicas.
3370
3630
  :param total_workers: Number of total workers.
@@ -3401,12 +3661,12 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
3401
3661
  session.execute(stmt)
3402
3662
 
3403
3663
  # Delete duplicates
3404
- if session.bind.dialect.name == 'oracle':
3664
+ if session.bind.dialect.name == 'oracle': # type: ignore
3405
3665
  schema = ''
3406
3666
  if BASE.metadata.schema:
3407
3667
  schema = BASE.metadata.schema + '.'
3408
3668
  session.execute(text('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))) # NOQA: E501
3409
- elif session.bind.dialect.name == 'mysql':
3669
+ elif session.bind.dialect.name == 'mysql': # type: ignore
3410
3670
  subquery1 = select(
3411
3671
  func.max(models.UpdatedCollectionReplica.id).label('max_id')
3412
3672
  ).group_by(
@@ -3464,7 +3724,11 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
3464
3724
 
3465
3725
 
3466
3726
  @transactional_session
3467
- def update_collection_replica(update_request, *, session: "Session"):
3727
+ def update_collection_replica(
3728
+ update_request: dict[str, Any],
3729
+ *,
3730
+ session: "Session"
3731
+ ) -> None:
3468
3732
  """
3469
3733
  Update a collection replica.
3470
3734
  :param update_request: update request from the upated_col_rep table.
@@ -3620,7 +3884,13 @@ def update_collection_replica(update_request, *, session: "Session"):
3620
3884
 
3621
3885
 
3622
3886
  @read_session
3623
- def get_bad_pfns(limit=10000, thread=None, total_threads=None, *, session: "Session"):
3887
+ def get_bad_pfns(
3888
+ limit: int = 10000,
3889
+ thread: Optional[int] = None,
3890
+ total_threads: Optional[int] = None,
3891
+ *,
3892
+ session: "Session"
3893
+ ) -> list[dict[str, Any]]:
3624
3894
  """
3625
3895
  Returns a list of bad PFNs
3626
3896
 
@@ -3653,7 +3923,15 @@ def get_bad_pfns(limit=10000, thread=None, total_threads=None, *, session: "Sess
3653
3923
 
3654
3924
 
3655
3925
  @transactional_session
3656
- def bulk_add_bad_replicas(replicas, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=None, expires_at=None, *, session: "Session"):
3926
+ def bulk_add_bad_replicas(
3927
+ replicas: "Iterable[dict[str, Any]]",
3928
+ account: InternalAccount,
3929
+ state: BadFilesStatus = BadFilesStatus.TEMPORARY_UNAVAILABLE,
3930
+ reason: Optional[str] = None,
3931
+ expires_at: Optional[datetime] = None,
3932
+ *,
3933
+ session: "Session"
3934
+ ) -> bool:
3657
3935
  """
3658
3936
  Bulk add new bad replicas.
3659
3937
 
@@ -3711,7 +3989,11 @@ def bulk_add_bad_replicas(replicas, account, state=BadFilesStatus.TEMPORARY_UNAV
3711
3989
 
3712
3990
 
3713
3991
  @transactional_session
3714
- def bulk_delete_bad_pfns(pfns, *, session: "Session"):
3992
+ def bulk_delete_bad_pfns(
3993
+ pfns: "Iterable[str]",
3994
+ *,
3995
+ session: "Session"
3996
+ ) -> Literal[True]:
3715
3997
  """
3716
3998
  Bulk delete bad PFNs.
3717
3999
 
@@ -3738,7 +4020,11 @@ def bulk_delete_bad_pfns(pfns, *, session: "Session"):
3738
4020
 
3739
4021
 
3740
4022
  @transactional_session
3741
- def bulk_delete_bad_replicas(bad_replicas, *, session: "Session"):
4023
+ def bulk_delete_bad_replicas(
4024
+ bad_replicas: "Iterable[dict[str, Any]]",
4025
+ *,
4026
+ session: "Session"
4027
+ ) -> Literal[True]:
3742
4028
  """
3743
4029
  Bulk delete bad replica.
3744
4030
 
@@ -3767,7 +4053,15 @@ def bulk_delete_bad_replicas(bad_replicas, *, session: "Session"):
3767
4053
 
3768
4054
 
3769
4055
  @transactional_session
3770
- def add_bad_pfns(pfns, account, state, reason=None, expires_at=None, *, session: "Session"):
4056
+ def add_bad_pfns(
4057
+ pfns: "Iterable[str]",
4058
+ account: InternalAccount,
4059
+ state: BadFilesStatus,
4060
+ reason: Optional[str] = None,
4061
+ expires_at: Optional[datetime] = None,
4062
+ *,
4063
+ session: "Session"
4064
+ ) -> Literal[True]:
3771
4065
  """
3772
4066
  Add bad PFNs.
3773
4067
 
@@ -3811,7 +4105,13 @@ def add_bad_pfns(pfns, account, state, reason=None, expires_at=None, *, session:
3811
4105
 
3812
4106
 
3813
4107
  @read_session
3814
- def list_expired_temporary_unavailable_replicas(total_workers, worker_number, limit=10000, *, session: "Session"):
4108
+ def list_expired_temporary_unavailable_replicas(
4109
+ total_workers: int,
4110
+ worker_number: int,
4111
+ limit: int = 10000,
4112
+ *,
4113
+ session: "Session"
4114
+ ) -> "Sequence[Row]":
3815
4115
  """
3816
4116
  List the expired temporary unavailable replicas
3817
4117
 
@@ -3843,7 +4143,12 @@ def list_expired_temporary_unavailable_replicas(total_workers, worker_number, li
3843
4143
 
3844
4144
 
3845
4145
  @read_session
3846
- def get_replicas_state(scope=None, name=None, *, session: "Session"):
4146
+ def get_replicas_state(
4147
+ scope: Optional[InternalScope] = None,
4148
+ name: Optional[str] = None,
4149
+ *,
4150
+ session: "Session"
4151
+ ) -> dict[ReplicaState, list[str]]:
3847
4152
  """
3848
4153
  Method used by the necromancer to get all the replicas of a DIDs
3849
4154
  :param scope: The scope of the file.
@@ -3873,16 +4178,16 @@ def get_replicas_state(scope=None, name=None, *, session: "Session"):
3873
4178
  def get_suspicious_files(
3874
4179
  rse_expression: str,
3875
4180
  available_elsewhere: int,
3876
- filter_: "Optional[dict[str, Any]]" = None,
4181
+ filter_: Optional[dict[str, Any]] = None,
3877
4182
  logger: "LoggerFunction" = logging.log,
3878
- younger_than: "Optional[datetime]" = None,
4183
+ younger_than: Optional[datetime] = None,
3879
4184
  nattempts: int = 0,
3880
4185
  nattempts_exact: bool = False,
3881
4186
  *,
3882
4187
  session: "Session",
3883
- exclude_states: "Optional[Iterable[str]]" = None,
4188
+ exclude_states: Optional["Iterable[str]"] = None,
3884
4189
  is_suspicious: bool = False
3885
- ) -> "list[dict[str, Any]]":
4190
+ ) -> list[dict[str, Any]]:
3886
4191
  """
3887
4192
  Gets a list of replicas from bad_replicas table which are: declared more than <nattempts> times since <younger_than> date,
3888
4193
  present on the RSE specified by the <rse_expression> and do not have a state in <exclude_states> list.
@@ -4025,7 +4330,15 @@ def get_suspicious_files(
4025
4330
 
4026
4331
 
4027
4332
  @read_session
4028
- def get_suspicious_reason(rse_id, scope, name, nattempts=0, logger=logging.log, *, session: "Session"):
4333
+ def get_suspicious_reason(
4334
+ rse_id: str,
4335
+ scope: InternalScope,
4336
+ name: str,
4337
+ nattempts: int = 0,
4338
+ logger: "LoggerFunction" = logging.log,
4339
+ *,
4340
+ session: "Session"
4341
+ ) -> list[dict[str, Any]]:
4029
4342
  """
4030
4343
  Returns the error message(s) which lead to the replica(s) being declared suspicious.
4031
4344
 
@@ -4085,7 +4398,14 @@ def get_suspicious_reason(rse_id, scope, name, nattempts=0, logger=logging.log,
4085
4398
 
4086
4399
 
4087
4400
  @transactional_session
4088
- def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, *, session: "Session"):
4401
+ def set_tombstone(
4402
+ rse_id: str,
4403
+ scope: InternalScope,
4404
+ name: str,
4405
+ tombstone: datetime = OBSOLETE,
4406
+ *,
4407
+ session: "Session"
4408
+ ) -> None:
4089
4409
  """
4090
4410
  Sets a tombstone on a replica.
4091
4411
 
@@ -4127,7 +4447,12 @@ def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, *, session: "Session"
4127
4447
 
4128
4448
 
4129
4449
  @read_session
4130
- def get_RSEcoverage_of_dataset(scope, name, *, session: "Session"):
4450
+ def get_rse_coverage_of_dataset(
4451
+ scope: "InternalScope",
4452
+ name: str,
4453
+ *,
4454
+ session: "Session"
4455
+ ) -> dict[str, int]:
4131
4456
  """
4132
4457
  Get total bytes present on RSEs
4133
4458