rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/{daemons/c3po/collectors → cli}/__init__.py +1 -0
- rucio/cli/account.py +216 -0
- rucio-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
- rucio-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/accountclient.py +0 -1
- rucio/client/baseclient.py +33 -24
- rucio/client/client.py +45 -1
- rucio/client/didclient.py +5 -3
- rucio/client/downloadclient.py +6 -8
- rucio/client/replicaclient.py +0 -2
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +4 -4
- rucio/client/uploadclient.py +26 -12
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +66 -29
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +22 -35
- rucio/common/constants.py +61 -3
- rucio/common/didtype.py +72 -24
- rucio/common/dumper/__init__.py +45 -38
- rucio/common/dumper/consistency.py +75 -30
- rucio/common/dumper/data_models.py +63 -19
- rucio/common/dumper/path_parsing.py +19 -8
- rucio/common/exception.py +65 -8
- rucio/common/extra.py +5 -10
- rucio/common/logging.py +13 -13
- rucio/common/pcache.py +8 -7
- rucio/common/plugins.py +59 -27
- rucio/common/policy.py +12 -3
- rucio/common/schema/__init__.py +84 -34
- rucio/common/schema/generic.py +0 -17
- rucio/common/schema/generic_multi_vo.py +0 -17
- rucio/common/stomp_utils.py +383 -119
- rucio/common/test_rucio_server.py +12 -6
- rucio/common/types.py +132 -52
- rucio/common/utils.py +93 -643
- rucio/core/account_limit.py +14 -12
- rucio/core/authentication.py +2 -2
- rucio/core/config.py +23 -42
- rucio/core/credential.py +14 -15
- rucio/core/did.py +5 -1
- rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
- rucio/core/did_meta_plugins/filter_engine.py +62 -3
- rucio/core/did_meta_plugins/json_meta.py +2 -2
- rucio/core/did_meta_plugins/mongo_meta.py +43 -30
- rucio/core/did_meta_plugins/postgres_meta.py +75 -39
- rucio/core/identity.py +6 -5
- rucio/core/importer.py +4 -3
- rucio/core/lifetime_exception.py +2 -2
- rucio/core/lock.py +8 -7
- rucio/core/message.py +6 -0
- rucio/core/monitor.py +30 -29
- rucio/core/naming_convention.py +2 -2
- rucio/core/nongrid_trace.py +2 -2
- rucio/core/oidc.py +11 -9
- rucio/core/permission/__init__.py +79 -37
- rucio/core/permission/generic.py +1 -7
- rucio/core/permission/generic_multi_vo.py +1 -7
- rucio/core/quarantined_replica.py +4 -3
- rucio/core/replica.py +464 -139
- rucio/core/replica_sorter.py +55 -59
- rucio/core/request.py +34 -32
- rucio/core/rse.py +301 -97
- rucio/core/rse_counter.py +1 -2
- rucio/core/rse_expression_parser.py +7 -7
- rucio/core/rse_selector.py +9 -7
- rucio/core/rule.py +41 -40
- rucio/core/rule_grouping.py +42 -40
- rucio/core/scope.py +5 -4
- rucio/core/subscription.py +26 -28
- rucio/core/topology.py +11 -11
- rucio/core/trace.py +2 -2
- rucio/core/transfer.py +29 -15
- rucio/core/volatile_replica.py +4 -3
- rucio/daemons/atropos/atropos.py +1 -1
- rucio/daemons/auditor/__init__.py +2 -2
- rucio/daemons/auditor/srmdumps.py +6 -6
- rucio/daemons/automatix/automatix.py +32 -21
- rucio/daemons/badreplicas/necromancer.py +2 -2
- rucio/daemons/bb8/nuclei_background_rebalance.py +1 -1
- rucio/daemons/bb8/t2_background_rebalance.py +1 -1
- rucio/daemons/cache/consumer.py +26 -90
- rucio/daemons/common.py +15 -25
- rucio/daemons/conveyor/finisher.py +2 -2
- rucio/daemons/conveyor/poller.py +18 -28
- rucio/daemons/conveyor/receiver.py +53 -123
- rucio/daemons/conveyor/stager.py +1 -0
- rucio/daemons/conveyor/submitter.py +3 -3
- rucio/daemons/hermes/hermes.py +129 -369
- rucio/daemons/judge/evaluator.py +2 -2
- rucio/daemons/oauthmanager/oauthmanager.py +3 -3
- rucio/daemons/reaper/dark_reaper.py +7 -3
- rucio/daemons/reaper/reaper.py +12 -16
- rucio/daemons/rsedecommissioner/config.py +1 -1
- rucio/daemons/rsedecommissioner/profiles/generic.py +5 -4
- rucio/daemons/rsedecommissioner/profiles/types.py +7 -6
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +1 -1
- rucio/daemons/storage/consistency/actions.py +8 -6
- rucio/daemons/tracer/kronos.py +117 -142
- rucio/db/sqla/constants.py +5 -0
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +4 -4
- rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
- rucio/db/sqla/models.py +157 -154
- rucio/db/sqla/session.py +58 -27
- rucio/db/sqla/types.py +2 -2
- rucio/db/sqla/util.py +2 -2
- rucio/gateway/account.py +18 -12
- rucio/gateway/account_limit.py +137 -60
- rucio/gateway/authentication.py +18 -12
- rucio/gateway/config.py +30 -20
- rucio/gateway/credential.py +9 -10
- rucio/gateway/did.py +70 -53
- rucio/gateway/dirac.py +6 -4
- rucio/gateway/exporter.py +3 -2
- rucio/gateway/heartbeat.py +6 -4
- rucio/gateway/identity.py +36 -51
- rucio/gateway/importer.py +3 -2
- rucio/gateway/lifetime_exception.py +3 -2
- rucio/gateway/meta_conventions.py +17 -6
- rucio/gateway/permission.py +4 -1
- rucio/gateway/quarantined_replica.py +3 -2
- rucio/gateway/replica.py +31 -22
- rucio/gateway/request.py +27 -18
- rucio/gateway/rse.py +69 -37
- rucio/gateway/rule.py +46 -26
- rucio/gateway/scope.py +3 -2
- rucio/gateway/subscription.py +14 -11
- rucio/gateway/vo.py +12 -8
- rucio/rse/__init__.py +3 -3
- rucio/rse/protocols/bittorrent.py +11 -1
- rucio/rse/protocols/cache.py +0 -11
- rucio/rse/protocols/dummy.py +0 -11
- rucio/rse/protocols/gfal.py +14 -9
- rucio/rse/protocols/globus.py +1 -1
- rucio/rse/protocols/http_cache.py +1 -1
- rucio/rse/protocols/posix.py +2 -2
- rucio/rse/protocols/protocol.py +84 -317
- rucio/rse/protocols/rclone.py +2 -1
- rucio/rse/protocols/rfio.py +10 -1
- rucio/rse/protocols/ssh.py +2 -1
- rucio/rse/protocols/storm.py +2 -13
- rucio/rse/protocols/webdav.py +74 -30
- rucio/rse/protocols/xrootd.py +2 -1
- rucio/rse/rsemanager.py +170 -53
- rucio/rse/translation.py +260 -0
- rucio/tests/common.py +23 -13
- rucio/tests/common_server.py +26 -9
- rucio/transfertool/bittorrent.py +15 -14
- rucio/transfertool/bittorrent_driver.py +5 -7
- rucio/transfertool/bittorrent_driver_qbittorrent.py +9 -8
- rucio/transfertool/fts3.py +20 -16
- rucio/transfertool/mock.py +2 -3
- rucio/vcsversion.py +4 -4
- rucio/version.py +7 -0
- rucio/web/rest/flaskapi/v1/accounts.py +17 -3
- rucio/web/rest/flaskapi/v1/auth.py +5 -5
- rucio/web/rest/flaskapi/v1/credentials.py +3 -2
- rucio/web/rest/flaskapi/v1/dids.py +21 -15
- rucio/web/rest/flaskapi/v1/identities.py +33 -9
- rucio/web/rest/flaskapi/v1/redirect.py +5 -4
- rucio/web/rest/flaskapi/v1/replicas.py +12 -8
- rucio/web/rest/flaskapi/v1/rses.py +15 -4
- rucio/web/rest/flaskapi/v1/traces.py +56 -19
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/alembic.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/alembic_offline.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio.cfg.atlas.client.template +3 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio.cfg.template +3 -19
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio_multi_vo.cfg.template +1 -18
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/requirements.server.txt +97 -68
- rucio-37.0.0rc2.data/scripts/rucio +133 -0
- rucio-37.0.0rc2.data/scripts/rucio-admin +97 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-atropos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-auditor +2 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-automatix +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-cache-client +17 -10
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-receiver +1 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-kronos +1 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-minos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-minos-temporary-expiration +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-necromancer +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-reaper +6 -6
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-transmogrifier +2 -2
- rucio-37.0.0rc2.dist-info/METADATA +92 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/RECORD +239 -245
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/atlas.py +0 -413
- rucio/common/schema/belleii.py +0 -408
- rucio/common/schema/domatpc.py +0 -401
- rucio/common/schema/escape.py +0 -426
- rucio/common/schema/icecube.py +0 -406
- rucio/core/permission/atlas.py +0 -1348
- rucio/core/permission/belleii.py +0 -1077
- rucio/core/permission/escape.py +0 -1078
- rucio/daemons/c3po/algorithms/__init__.py +0 -13
- rucio/daemons/c3po/algorithms/simple.py +0 -134
- rucio/daemons/c3po/algorithms/t2_free_space.py +0 -128
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +0 -130
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +0 -294
- rucio/daemons/c3po/c3po.py +0 -371
- rucio/daemons/c3po/collectors/agis.py +0 -108
- rucio/daemons/c3po/collectors/free_space.py +0 -81
- rucio/daemons/c3po/collectors/jedi_did.py +0 -57
- rucio/daemons/c3po/collectors/mock_did.py +0 -51
- rucio/daemons/c3po/collectors/network_metrics.py +0 -71
- rucio/daemons/c3po/collectors/workload.py +0 -112
- rucio/daemons/c3po/utils/__init__.py +0 -13
- rucio/daemons/c3po/utils/dataset_cache.py +0 -50
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +0 -56
- rucio/daemons/c3po/utils/expiring_list.py +0 -62
- rucio/daemons/c3po/utils/popularity.py +0 -85
- rucio/daemons/c3po/utils/timeseries.py +0 -89
- rucio/rse/protocols/gsiftp.py +0 -92
- rucio-35.7.0.data/scripts/rucio-c3po +0 -85
- rucio-35.7.0.dist-info/METADATA +0 -72
- /rucio/{daemons/c3po → cli/bin_legacy}/__init__.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-account +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-bb8 +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-dumper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-follower +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-hermes +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-injector +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-undertaker +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/WHEEL +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/licenses/LICENSE +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -16,8 +16,9 @@ import json
|
|
|
16
16
|
import operator
|
|
17
17
|
from typing import TYPE_CHECKING
|
|
18
18
|
|
|
19
|
-
import
|
|
20
|
-
import
|
|
19
|
+
import psycopg
|
|
20
|
+
from psycopg import sql
|
|
21
|
+
from psycopg.rows import dict_row
|
|
21
22
|
|
|
22
23
|
from rucio.common import config, exception
|
|
23
24
|
from rucio.common.types import InternalScope
|
|
@@ -73,17 +74,20 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
73
74
|
self.jsonb_column = table_column_data
|
|
74
75
|
|
|
75
76
|
self.table = table
|
|
76
|
-
self.client =
|
|
77
|
+
self.client = psycopg.connect(
|
|
77
78
|
host=host,
|
|
78
79
|
port=port,
|
|
79
|
-
|
|
80
|
+
dbname=db,
|
|
80
81
|
user=user,
|
|
81
82
|
password=password)
|
|
82
83
|
|
|
83
84
|
# set search_path to include database schema by default
|
|
84
85
|
cur = self.client.cursor()
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
cur.execute(
|
|
87
|
+
sql.SQL("SET search_path TO {}").format(
|
|
88
|
+
sql.Identifier(db_schema)
|
|
89
|
+
)
|
|
90
|
+
)
|
|
87
91
|
cur.close()
|
|
88
92
|
|
|
89
93
|
if not table_is_managed: # not managed by Rucio, so just verify table schema
|
|
@@ -97,17 +101,18 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
97
101
|
"""
|
|
98
102
|
Try to create a metadata table.
|
|
99
103
|
"""
|
|
100
|
-
table_clauses =
|
|
101
|
-
("id
|
|
102
|
-
("vo
|
|
103
|
-
("scope
|
|
104
|
-
("name
|
|
105
|
-
("data
|
|
106
|
-
("UNIQUE
|
|
104
|
+
table_clauses = [
|
|
105
|
+
sql.SQL("id bigint NOT NULL GENERATED ALWAYS AS IDENTITY"),
|
|
106
|
+
sql.SQL("vo varchar NOT NULL"),
|
|
107
|
+
sql.SQL("scope varchar NOT NULL"),
|
|
108
|
+
sql.SQL("name varchar NOT NULL"),
|
|
109
|
+
sql.SQL("data jsonb DEFAULT '{}'::jsonb"),
|
|
110
|
+
sql.SQL("UNIQUE (scope, name)") # unique scope+name table constraint, required for ON CONFLICT
|
|
111
|
+
]
|
|
112
|
+
statement = sql.SQL("CREATE TABLE IF NOT EXISTS {} ({})").format(
|
|
113
|
+
sql.Identifier(self.table),
|
|
114
|
+
sql.SQL(', ').join(table_clauses)
|
|
107
115
|
)
|
|
108
|
-
statement = "CREATE TABLE IF NOT EXISTS {} ({})".format(
|
|
109
|
-
self.table,
|
|
110
|
-
', '.join([' '.join(clause) for clause in table_clauses]))
|
|
111
116
|
|
|
112
117
|
cur = self.client.cursor()
|
|
113
118
|
cur.execute(statement)
|
|
@@ -127,8 +132,11 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
127
132
|
:raises: MetadataSchemaMismatchError
|
|
128
133
|
"""
|
|
129
134
|
# Check mandatory columns are of right data type and have the right nullable qualifier.
|
|
130
|
-
statement =
|
|
131
|
-
|
|
135
|
+
statement = sql.SQL(
|
|
136
|
+
"SELECT column_name, data_type, is_nullable FROM INFORMATION_SCHEMA.COLUMNS where table_name = {}").format(
|
|
137
|
+
sql.Literal(self.table)
|
|
138
|
+
)
|
|
139
|
+
|
|
132
140
|
cur = self.client.cursor()
|
|
133
141
|
cur.execute(statement)
|
|
134
142
|
existing_table_columns = cur.fetchall()
|
|
@@ -147,14 +155,19 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
147
155
|
specification, existing_table_columns))
|
|
148
156
|
|
|
149
157
|
# Check required table constraints exist.
|
|
150
|
-
statement =
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
+
statement = sql.SQL(
|
|
159
|
+
"SELECT con.contype AS constraint_type, " # type: ignore
|
|
160
|
+
"(SELECT array_agg(att.attname) FROM pg_attribute att "
|
|
161
|
+
" INNER JOIN unnest(con.conkey) unnest(conkey) ON unnest.conkey = att.attnum "
|
|
162
|
+
" WHERE att.attrelid = con.conrelid) AS columns "
|
|
163
|
+
"FROM pg_constraint con "
|
|
164
|
+
"INNER JOIN pg_class rel ON rel.oid = con.conrelid "
|
|
165
|
+
"INNER JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace "
|
|
166
|
+
"WHERE rel.relname = {}"
|
|
167
|
+
).format(
|
|
168
|
+
sql.Literal(self.table)
|
|
169
|
+
)
|
|
170
|
+
|
|
158
171
|
cur = self.client.cursor()
|
|
159
172
|
cur.execute(statement)
|
|
160
173
|
existing_table_constraints = cur.fetchall() # list of (constraint_type, [columns])
|
|
@@ -170,9 +183,12 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
170
183
|
constraint, len(existing_table_constraints)))
|
|
171
184
|
|
|
172
185
|
def _drop_metadata_table(self):
|
|
173
|
-
statement = "DROP TABLE IF EXISTS {};".format(self.table)
|
|
174
186
|
cur = self.client.cursor()
|
|
175
|
-
cur.execute(
|
|
187
|
+
cur.execute(
|
|
188
|
+
sql.SQL("DROP TABLE IF EXISTS {}").format(
|
|
189
|
+
sql.Identifier(self.table)
|
|
190
|
+
)
|
|
191
|
+
)
|
|
176
192
|
cur.close()
|
|
177
193
|
self.client.commit()
|
|
178
194
|
|
|
@@ -185,8 +201,12 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
185
201
|
:param session: The database session in use
|
|
186
202
|
:returns: the metadata for the did
|
|
187
203
|
"""
|
|
188
|
-
statement = "SELECT data from {} ".format(
|
|
189
|
-
|
|
204
|
+
statement = sql.SQL("SELECT data from {} WHERE scope = {} AND name = {}").format(
|
|
205
|
+
sql.Identifier(self.table),
|
|
206
|
+
sql.Literal(scope.internal),
|
|
207
|
+
sql.Literal(name)
|
|
208
|
+
)
|
|
209
|
+
|
|
190
210
|
cur = self.client.cursor()
|
|
191
211
|
cur.execute(statement)
|
|
192
212
|
metadata = cur.fetchone()
|
|
@@ -221,9 +241,18 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
221
241
|
:param session: The database session in use
|
|
222
242
|
"""
|
|
223
243
|
# upsert metadata
|
|
224
|
-
statement =
|
|
225
|
-
|
|
226
|
-
|
|
244
|
+
statement = sql.SQL(
|
|
245
|
+
"INSERT INTO {} (scope, name, vo, data) VALUES ({}, {}, {}, {}) " # type: ignore
|
|
246
|
+
"ON CONFLICT (scope, name) DO UPDATE set data = {}.data || EXCLUDED.data"
|
|
247
|
+
).format(
|
|
248
|
+
sql.Identifier(self.table),
|
|
249
|
+
sql.Literal(scope.external),
|
|
250
|
+
sql.Literal(name),
|
|
251
|
+
sql.Literal(scope.vo),
|
|
252
|
+
sql.Literal(json.dumps(metadata)),
|
|
253
|
+
sql.Identifier(self.table)
|
|
254
|
+
)
|
|
255
|
+
|
|
227
256
|
cur = self.client.cursor()
|
|
228
257
|
cur.execute(statement)
|
|
229
258
|
cur.close()
|
|
@@ -238,8 +267,12 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
238
267
|
:param key: the key to be deleted
|
|
239
268
|
:param session: the database session in use
|
|
240
269
|
"""
|
|
241
|
-
statement = "UPDATE {} ".format(
|
|
242
|
-
|
|
270
|
+
statement = sql.SQL("UPDATE {} SET data = {}.data - {}").format(
|
|
271
|
+
sql.Identifier(self.table),
|
|
272
|
+
sql.Identifier(self.table),
|
|
273
|
+
sql.Literal(key)
|
|
274
|
+
)
|
|
275
|
+
|
|
243
276
|
cur = self.client.cursor()
|
|
244
277
|
cur.execute(statement)
|
|
245
278
|
cur.close()
|
|
@@ -276,10 +309,13 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
|
|
|
276
309
|
"'{}' metadata module does not currently support recursive searches".format(self.plugin_name.lower())
|
|
277
310
|
)
|
|
278
311
|
|
|
279
|
-
statement = "SELECT * FROM {} WHERE {} ".format(
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
312
|
+
statement = sql.SQL("SELECT * FROM {} WHERE {} {}").format(
|
|
313
|
+
sql.Identifier(self.table),
|
|
314
|
+
sql.SQL(postgres_query_str), # type: ignore
|
|
315
|
+
sql.SQL("LIMIT {}").format(sql.Literal(limit)) if limit else sql.SQL("")
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
cur = self.client.cursor(row_factory=dict_row)
|
|
283
319
|
cur.execute(statement)
|
|
284
320
|
query_result = cur.fetchall()
|
|
285
321
|
cur.close()
|
rucio/core/identity.py
CHANGED
|
@@ -21,7 +21,6 @@ from sqlalchemy import select, true
|
|
|
21
21
|
from sqlalchemy.exc import IntegrityError
|
|
22
22
|
|
|
23
23
|
from rucio.common import exception
|
|
24
|
-
from rucio.common.types import InternalAccount
|
|
25
24
|
from rucio.core.account import account_exists
|
|
26
25
|
from rucio.db.sqla import models
|
|
27
26
|
from rucio.db.sqla.constants import IdentityType
|
|
@@ -33,6 +32,8 @@ if TYPE_CHECKING:
|
|
|
33
32
|
from sqlalchemy import Row
|
|
34
33
|
from sqlalchemy.orm import Session
|
|
35
34
|
|
|
35
|
+
from rucio.common.types import InternalAccount
|
|
36
|
+
|
|
36
37
|
|
|
37
38
|
@transactional_session
|
|
38
39
|
def add_identity(identity: str, type_: IdentityType, email: str, password: Optional[str] = None, *, session: "Session") -> None:
|
|
@@ -129,7 +130,7 @@ def del_identity(identity: str, type_: IdentityType, *, session: "Session") -> N
|
|
|
129
130
|
def add_account_identity(
|
|
130
131
|
identity: str,
|
|
131
132
|
type_: IdentityType,
|
|
132
|
-
account: InternalAccount,
|
|
133
|
+
account: "InternalAccount",
|
|
133
134
|
email: str,
|
|
134
135
|
default: bool = False,
|
|
135
136
|
password: Optional[str] = None,
|
|
@@ -177,7 +178,7 @@ def add_account_identity(
|
|
|
177
178
|
|
|
178
179
|
|
|
179
180
|
@read_session
|
|
180
|
-
def exist_identity_account(identity: str, type_: IdentityType, account: InternalAccount, *, session: "Session") -> bool:
|
|
181
|
+
def exist_identity_account(identity: str, type_: IdentityType, account: "InternalAccount", *, session: "Session") -> bool:
|
|
181
182
|
"""
|
|
182
183
|
Check if an identity is mapped to an account.
|
|
183
184
|
|
|
@@ -199,7 +200,7 @@ def exist_identity_account(identity: str, type_: IdentityType, account: Internal
|
|
|
199
200
|
|
|
200
201
|
|
|
201
202
|
@read_session
|
|
202
|
-
def get_default_account(identity: str, type_: IdentityType, oldest_if_none: bool = False, *, session: "Session") -> Optional[InternalAccount]:
|
|
203
|
+
def get_default_account(identity: str, type_: IdentityType, oldest_if_none: bool = False, *, session: "Session") -> Optional["InternalAccount"]:
|
|
203
204
|
"""
|
|
204
205
|
Retrieves the default account mapped to an identity.
|
|
205
206
|
|
|
@@ -240,7 +241,7 @@ def get_default_account(identity: str, type_: IdentityType, oldest_if_none: bool
|
|
|
240
241
|
|
|
241
242
|
|
|
242
243
|
@transactional_session
|
|
243
|
-
def del_account_identity(identity: str, type_: IdentityType, account: InternalAccount, *, session: "Session") -> None:
|
|
244
|
+
def del_account_identity(identity: str, type_: IdentityType, account: "InternalAccount", *, session: "Session") -> None:
|
|
244
245
|
"""
|
|
245
246
|
Removes a membership association between identity and account.
|
|
246
247
|
|
rucio/core/importer.py
CHANGED
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from collections.abc import Iterable
|
|
16
15
|
from typing import TYPE_CHECKING, Any
|
|
17
16
|
|
|
18
17
|
from sqlalchemy import select
|
|
@@ -30,6 +29,8 @@ from rucio.db.sqla.constants import AccountType, IdentityType, RSEType
|
|
|
30
29
|
from rucio.db.sqla.session import transactional_session
|
|
31
30
|
|
|
32
31
|
if TYPE_CHECKING:
|
|
32
|
+
from collections.abc import Iterable
|
|
33
|
+
|
|
33
34
|
from sqlalchemy.orm import Session
|
|
34
35
|
|
|
35
36
|
|
|
@@ -162,7 +163,7 @@ def import_distances(distances, vo: str = 'def', *, session: "Session") -> None:
|
|
|
162
163
|
|
|
163
164
|
|
|
164
165
|
@transactional_session
|
|
165
|
-
def import_identities(identities: Iterable[dict[str, Any]], account_name: str, old_identities: Iterable[tuple], old_identity_account: tuple[str, str, str], account_email: str, *, session: "Session") -> None:
|
|
166
|
+
def import_identities(identities: 'Iterable[dict[str, Any]]', account_name: str, old_identities: 'Iterable[tuple]', old_identity_account: tuple[str, str, str], account_email: str, *, session: "Session") -> None:
|
|
166
167
|
for identity in identities:
|
|
167
168
|
identity['type'] = IdentityType[identity['type'].upper()]
|
|
168
169
|
|
|
@@ -191,7 +192,7 @@ def import_identities(identities: Iterable[dict[str, Any]], account_name: str, o
|
|
|
191
192
|
|
|
192
193
|
|
|
193
194
|
@transactional_session
|
|
194
|
-
def import_accounts(accounts: Iterable[dict[str, Any]], vo: str = 'def', *, session: "Session") -> None:
|
|
195
|
+
def import_accounts(accounts: 'Iterable[dict[str, Any]]', vo: str = 'def', *, session: "Session") -> None:
|
|
195
196
|
vo_filter = {'account': InternalAccount(account='*', vo=vo)}
|
|
196
197
|
old_accounts = {account['account']: account for account in account_module.list_accounts(filter_=vo_filter, session=session)}
|
|
197
198
|
missing_accounts = [account for account in accounts if account['account'] not in old_accounts]
|
rucio/core/lifetime_exception.py
CHANGED
|
@@ -245,8 +245,8 @@ def update_exception(
|
|
|
245
245
|
:param state: The states to filter
|
|
246
246
|
:param session: The database session in use.
|
|
247
247
|
"""
|
|
248
|
-
|
|
249
|
-
if state not in
|
|
248
|
+
allowed_states = (LifetimeExceptionsState.APPROVED, LifetimeExceptionsState.REJECTED)
|
|
249
|
+
if state not in allowed_states:
|
|
250
250
|
raise UnsupportedOperation
|
|
251
251
|
|
|
252
252
|
query = update(
|
rucio/core/lock.py
CHANGED
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import logging
|
|
16
|
-
from collections.abc import Iterable, Iterator
|
|
17
16
|
from datetime import datetime
|
|
18
17
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
19
18
|
|
|
@@ -32,11 +31,13 @@ from rucio.db.sqla.constants import DIDType, LockState, RuleGrouping, RuleNotifi
|
|
|
32
31
|
from rucio.db.sqla.session import read_session, stream_session, transactional_session
|
|
33
32
|
|
|
34
33
|
if TYPE_CHECKING:
|
|
34
|
+
from collections.abc import Iterable, Iterator
|
|
35
|
+
|
|
35
36
|
from sqlalchemy.orm import Session
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
@stream_session
|
|
39
|
-
def get_dataset_locks(scope: InternalScope, name: str, *, session: "Session") -> Iterator[dict[str, Any]]:
|
|
40
|
+
def get_dataset_locks(scope: InternalScope, name: str, *, session: "Session") -> "Iterator[dict[str, Any]]":
|
|
40
41
|
"""
|
|
41
42
|
Get the dataset locks of a dataset
|
|
42
43
|
|
|
@@ -75,7 +76,7 @@ def get_dataset_locks(scope: InternalScope, name: str, *, session: "Session") ->
|
|
|
75
76
|
|
|
76
77
|
|
|
77
78
|
@stream_session
|
|
78
|
-
def get_dataset_locks_bulk(dids: Iterable[dict[str, Any]], *, session: "Session") -> Iterator[dict[str, Any]]:
|
|
79
|
+
def get_dataset_locks_bulk(dids: "Iterable[dict[str, Any]]", *, session: "Session") -> "Iterator[dict[str, Any]]":
|
|
79
80
|
"""
|
|
80
81
|
Get the dataset locks of a list of datasets or containers, recursively
|
|
81
82
|
|
|
@@ -111,7 +112,7 @@ def get_dataset_locks_bulk(dids: Iterable[dict[str, Any]], *, session: "Session"
|
|
|
111
112
|
|
|
112
113
|
|
|
113
114
|
@stream_session
|
|
114
|
-
def get_dataset_locks_by_rse_id(rse_id: str, *, session: "Session") -> Iterator[dict[str, Any]]:
|
|
115
|
+
def get_dataset_locks_by_rse_id(rse_id: str, *, session: "Session") -> "Iterator[dict[str, Any]]":
|
|
115
116
|
"""
|
|
116
117
|
Get the dataset locks of an RSE.
|
|
117
118
|
|
|
@@ -151,7 +152,7 @@ def get_dataset_locks_by_rse_id(rse_id: str, *, session: "Session") -> Iterator[
|
|
|
151
152
|
|
|
152
153
|
|
|
153
154
|
@read_session
|
|
154
|
-
def get_replica_locks(scope: InternalScope, name: str, nowait: bool = False, restrict_rses: Optional[Iterable[str]] = None, *, session: "Session") -> list[models.ReplicaLock]:
|
|
155
|
+
def get_replica_locks(scope: InternalScope, name: str, nowait: bool = False, restrict_rses: Optional["Iterable[str]"] = None, *, session: "Session") -> list[models.ReplicaLock]:
|
|
155
156
|
"""
|
|
156
157
|
Get the active replica locks for a file
|
|
157
158
|
|
|
@@ -244,7 +245,7 @@ def get_replica_locks_for_rule_id_per_rse(rule_id: str, *, session: "Session") -
|
|
|
244
245
|
|
|
245
246
|
|
|
246
247
|
@read_session
|
|
247
|
-
def get_files_and_replica_locks_of_dataset(scope: InternalScope, name: str, nowait: bool = False, restrict_rses: Optional[Iterable[str]] = None, only_stuck: bool = False,
|
|
248
|
+
def get_files_and_replica_locks_of_dataset(scope: InternalScope, name: str, nowait: bool = False, restrict_rses: Optional["Iterable[str]"] = None, only_stuck: bool = False,
|
|
248
249
|
total_threads: Optional[int] = None, thread_id: Optional[int] = None,
|
|
249
250
|
*, session: "Session") -> dict[tuple[InternalScope, str], Union[models.ReplicaLock, list[models.ReplicaLock]]]:
|
|
250
251
|
"""
|
|
@@ -527,7 +528,7 @@ def failed_transfer(scope: InternalScope, name: str, rse_id: str, error_message:
|
|
|
527
528
|
|
|
528
529
|
|
|
529
530
|
@transactional_session
|
|
530
|
-
def touch_dataset_locks(dataset_locks: Iterable[dict[str, Any]], *, session: "Session") -> bool:
|
|
531
|
+
def touch_dataset_locks(dataset_locks: "Iterable[dict[str, Any]]", *, session: "Session") -> bool:
|
|
531
532
|
"""
|
|
532
533
|
Update the accessed_at timestamp of the given dataset locks + eol_at.
|
|
533
534
|
|
rucio/core/message.py
CHANGED
|
@@ -101,6 +101,7 @@ def retrieve_messages(bulk: int = 1000,
|
|
|
101
101
|
event_type: "Optional[str]" = None,
|
|
102
102
|
lock: bool = False,
|
|
103
103
|
old_mode: bool = True,
|
|
104
|
+
service_filter: "Optional[str]" = None,
|
|
104
105
|
*, session: "Session") -> "MessagesListType":
|
|
105
106
|
"""
|
|
106
107
|
Retrieve up to $bulk messages.
|
|
@@ -112,6 +113,7 @@ def retrieve_messages(bulk: int = 1000,
|
|
|
112
113
|
:param lock: Select exclusively some rows.
|
|
113
114
|
:param old_mode: If True, doesn't return email if event_type is None.
|
|
114
115
|
:param session: The database session to use.
|
|
116
|
+
:param service_filter: When a service is supplied this queries the database for messages for that service.
|
|
115
117
|
|
|
116
118
|
:returns messages: List of dictionaries {id, created_at, event_type, payload, services}
|
|
117
119
|
"""
|
|
@@ -123,6 +125,10 @@ def retrieve_messages(bulk: int = 1000,
|
|
|
123
125
|
Message.created_at
|
|
124
126
|
)
|
|
125
127
|
stmt_subquery = filter_thread_work(session=session, query=stmt_subquery, total_threads=total_threads, thread_id=thread)
|
|
128
|
+
if service_filter:
|
|
129
|
+
stmt_subquery = stmt_subquery.where(
|
|
130
|
+
Message.services == service_filter
|
|
131
|
+
)
|
|
126
132
|
if event_type:
|
|
127
133
|
stmt_subquery = stmt_subquery.where(
|
|
128
134
|
Message.event_type == event_type
|
rucio/core/monitor.py
CHANGED
|
@@ -21,7 +21,6 @@ import logging
|
|
|
21
21
|
import os
|
|
22
22
|
import string
|
|
23
23
|
from abc import abstractmethod
|
|
24
|
-
from collections.abc import Callable, Iterable, Sequence
|
|
25
24
|
from datetime import datetime, timedelta
|
|
26
25
|
from functools import wraps
|
|
27
26
|
from pathlib import Path
|
|
@@ -37,6 +36,8 @@ from rucio.common.stopwatch import Stopwatch
|
|
|
37
36
|
from rucio.common.utils import retrying
|
|
38
37
|
|
|
39
38
|
if TYPE_CHECKING:
|
|
39
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
40
|
+
|
|
40
41
|
from rucio.common.types import LoggerFunction
|
|
41
42
|
|
|
42
43
|
_T = TypeVar('_T')
|
|
@@ -61,15 +62,15 @@ class MultiprocessMutexValue(values.MultiProcessValue()):
|
|
|
61
62
|
super().__init__(*args, **kwargs)
|
|
62
63
|
self._lock = Lock()
|
|
63
64
|
|
|
64
|
-
def inc(self, amount):
|
|
65
|
+
def inc(self, amount: float) -> None:
|
|
65
66
|
with self._lock:
|
|
66
67
|
return super().inc(amount)
|
|
67
68
|
|
|
68
|
-
def set(self, value):
|
|
69
|
+
def set(self, value: float) -> None:
|
|
69
70
|
with self._lock:
|
|
70
71
|
return super().set(value)
|
|
71
72
|
|
|
72
|
-
def get(self):
|
|
73
|
+
def get(self) -> float:
|
|
73
74
|
with self._lock:
|
|
74
75
|
return super().get()
|
|
75
76
|
|
|
@@ -163,7 +164,7 @@ class _MultiMetric:
|
|
|
163
164
|
statsd: str,
|
|
164
165
|
prom: Optional[Union[str, Counter, Gauge, Histogram]] = None,
|
|
165
166
|
documentation: Optional[str] = None,
|
|
166
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
167
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
167
168
|
registry: Optional[CollectorRegistry] = None
|
|
168
169
|
):
|
|
169
170
|
"""
|
|
@@ -191,7 +192,7 @@ class _MultiMetric:
|
|
|
191
192
|
self._labelnames = labelnames
|
|
192
193
|
|
|
193
194
|
@abstractmethod
|
|
194
|
-
def init_prometheus_metric(self, name: str, documentation: Optional[str], labelnames: Sequence[str] = ()):
|
|
195
|
+
def init_prometheus_metric(self, name: str, documentation: Optional[str], labelnames: 'Sequence[str]' = ()):
|
|
195
196
|
pass
|
|
196
197
|
|
|
197
198
|
def labels(self: _M, **labelkwargs) -> _M:
|
|
@@ -199,7 +200,7 @@ class _MultiMetric:
|
|
|
199
200
|
return self
|
|
200
201
|
|
|
201
202
|
return self.__class__(
|
|
202
|
-
prom=self._prom.labels(**labelkwargs),
|
|
203
|
+
prom=self._prom.labels(**labelkwargs), # type: ignore
|
|
203
204
|
statsd=self._statsd.format(**labelkwargs),
|
|
204
205
|
documentation=self._documentation,
|
|
205
206
|
labelnames=self._labelnames,
|
|
@@ -209,24 +210,24 @@ class _MultiMetric:
|
|
|
209
210
|
|
|
210
211
|
class _MultiCounter(_MultiMetric):
|
|
211
212
|
|
|
212
|
-
def inc(self, delta=1):
|
|
213
|
+
def inc(self, delta: int = 1) -> None:
|
|
213
214
|
delta = abs(delta)
|
|
214
|
-
self._prom.inc(delta)
|
|
215
|
+
self._prom.inc(delta) # type: ignore
|
|
215
216
|
if STATSD_CLIENT:
|
|
216
217
|
STATSD_CLIENT.incr(self._statsd, delta)
|
|
217
218
|
|
|
218
|
-
def init_prometheus_metric(self, name: str, documentation: str, labelnames: Sequence[str] = ()) -> Counter:
|
|
219
|
+
def init_prometheus_metric(self, name: str, documentation: str, labelnames: 'Sequence[str]' = ()) -> Counter:
|
|
219
220
|
return Counter(name, documentation, labelnames=labelnames, registry=self._registry)
|
|
220
221
|
|
|
221
222
|
|
|
222
223
|
class _MultiGauge(_MultiMetric):
|
|
223
224
|
|
|
224
|
-
def set(self, value):
|
|
225
|
-
self._prom.set(value)
|
|
225
|
+
def set(self, value) -> None:
|
|
226
|
+
self._prom.set(value) # type: ignore
|
|
226
227
|
if STATSD_CLIENT:
|
|
227
228
|
STATSD_CLIENT.gauge(self._statsd, value)
|
|
228
229
|
|
|
229
|
-
def init_prometheus_metric(self, name: str, documentation: str, labelnames: Sequence[str] = ()) -> Gauge:
|
|
230
|
+
def init_prometheus_metric(self, name: str, documentation: str, labelnames: 'Sequence[str]' = ()) -> Gauge:
|
|
230
231
|
return Gauge(name, documentation, labelnames=labelnames, registry=self._registry)
|
|
231
232
|
|
|
232
233
|
|
|
@@ -237,20 +238,20 @@ class _MultiTiming(_MultiMetric):
|
|
|
237
238
|
statsd: str,
|
|
238
239
|
prom: Optional[str] = None,
|
|
239
240
|
documentation: Optional[str] = None,
|
|
240
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
241
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
241
242
|
registry: Optional[CollectorRegistry] = None,
|
|
242
|
-
buckets: Iterable[float] = _HISTOGRAM_DEFAULT_BUCKETS,
|
|
243
|
+
buckets: 'Iterable[float]' = _HISTOGRAM_DEFAULT_BUCKETS,
|
|
243
244
|
) -> None:
|
|
244
245
|
self._stopwatch = None
|
|
245
246
|
self._histogram_buckets = tuple(buckets)
|
|
246
247
|
super().__init__(statsd, prom, documentation, labelnames, registry)
|
|
247
248
|
|
|
248
|
-
def observe(self, value: float):
|
|
249
|
-
self._prom.observe(value)
|
|
249
|
+
def observe(self, value: float) -> None:
|
|
250
|
+
self._prom.observe(value) # type: ignore
|
|
250
251
|
if STATSD_CLIENT:
|
|
251
252
|
STATSD_CLIENT.timing(self._statsd, value * 1000)
|
|
252
253
|
|
|
253
|
-
def init_prometheus_metric(self, name: str, documentation: str, labelnames: Sequence[str] = ()) -> Histogram:
|
|
254
|
+
def init_prometheus_metric(self, name: str, documentation: str, labelnames: 'Sequence[str]' = ()) -> Histogram:
|
|
254
255
|
return Histogram(name, documentation, labelnames=labelnames, registry=self._registry, buckets=self._histogram_buckets)
|
|
255
256
|
|
|
256
257
|
def __enter__(self):
|
|
@@ -265,9 +266,9 @@ class _MultiTiming(_MultiMetric):
|
|
|
265
266
|
|
|
266
267
|
def _fetch_or_create_metric(
|
|
267
268
|
name: str,
|
|
268
|
-
labelnames: Optional[Sequence[str]],
|
|
269
|
+
labelnames: Optional['Sequence[str]'],
|
|
269
270
|
container: dict[str, _T],
|
|
270
|
-
factory: Callable[[str, Optional[Sequence[str]]], _T]
|
|
271
|
+
factory: 'Callable[[str, Optional[Sequence[str]]], _T]'
|
|
271
272
|
) -> "_T":
|
|
272
273
|
metric = container.get(name)
|
|
273
274
|
if not metric:
|
|
@@ -280,7 +281,7 @@ def _fetch_or_create_metric(
|
|
|
280
281
|
|
|
281
282
|
def _fetch_or_create_counter(
|
|
282
283
|
name: str,
|
|
283
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
284
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
284
285
|
documentation: Optional[str] = None,
|
|
285
286
|
registry: Optional[CollectorRegistry] = None,
|
|
286
287
|
) -> _MultiCounter:
|
|
@@ -295,7 +296,7 @@ def _fetch_or_create_counter(
|
|
|
295
296
|
|
|
296
297
|
def _fetch_or_create_gauge(
|
|
297
298
|
name: str,
|
|
298
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
299
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
299
300
|
documentation: Optional[str] = None,
|
|
300
301
|
registry: Optional[CollectorRegistry] = None,
|
|
301
302
|
) -> _MultiGauge:
|
|
@@ -310,10 +311,10 @@ def _fetch_or_create_gauge(
|
|
|
310
311
|
|
|
311
312
|
def _fetch_or_create_timer(
|
|
312
313
|
name: str,
|
|
313
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
314
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
314
315
|
documentation: Optional[str] = None,
|
|
315
316
|
registry: Optional[CollectorRegistry] = None,
|
|
316
|
-
buckets: Iterable[float] = _HISTOGRAM_DEFAULT_BUCKETS
|
|
317
|
+
buckets: 'Iterable[float]' = _HISTOGRAM_DEFAULT_BUCKETS
|
|
317
318
|
) -> _MultiTiming:
|
|
318
319
|
return _fetch_or_create_metric(
|
|
319
320
|
name=name,
|
|
@@ -332,7 +333,7 @@ class MetricManager:
|
|
|
332
333
|
"""
|
|
333
334
|
|
|
334
335
|
def __init__(self, prefix: Optional[str] = None, module: Optional[str] = None,
|
|
335
|
-
registry: Optional[CollectorRegistry] = None, push_gateways: Optional[Sequence[str]] = None):
|
|
336
|
+
registry: Optional[CollectorRegistry] = None, push_gateways: Optional['Sequence[str]'] = None):
|
|
336
337
|
if prefix:
|
|
337
338
|
self.prefix = prefix
|
|
338
339
|
elif module:
|
|
@@ -354,7 +355,7 @@ class MetricManager:
|
|
|
354
355
|
self,
|
|
355
356
|
name: str,
|
|
356
357
|
*,
|
|
357
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
358
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
358
359
|
documentation: Optional[str] = None,
|
|
359
360
|
) -> _MultiCounter:
|
|
360
361
|
"""
|
|
@@ -370,7 +371,7 @@ class MetricManager:
|
|
|
370
371
|
self,
|
|
371
372
|
name: str,
|
|
372
373
|
*,
|
|
373
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
374
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
374
375
|
documentation: Optional[str] = None,
|
|
375
376
|
) -> _MultiGauge:
|
|
376
377
|
"""
|
|
@@ -386,9 +387,9 @@ class MetricManager:
|
|
|
386
387
|
self,
|
|
387
388
|
name: str,
|
|
388
389
|
*,
|
|
389
|
-
labelnames: Optional[Sequence[str]] = None,
|
|
390
|
+
labelnames: Optional['Sequence[str]'] = None,
|
|
390
391
|
documentation: Optional[str] = None,
|
|
391
|
-
buckets: Iterable[float] = _HISTOGRAM_DEFAULT_BUCKETS
|
|
392
|
+
buckets: 'Iterable[float]' = _HISTOGRAM_DEFAULT_BUCKETS
|
|
392
393
|
) -> _MultiTiming:
|
|
393
394
|
"""
|
|
394
395
|
Log a time measurement.
|
rucio/core/naming_convention.py
CHANGED
|
@@ -20,7 +20,7 @@ from dogpile.cache.api import NO_VALUE
|
|
|
20
20
|
from sqlalchemy import and_, delete, select
|
|
21
21
|
from sqlalchemy.exc import IntegrityError
|
|
22
22
|
|
|
23
|
-
from rucio.common.cache import
|
|
23
|
+
from rucio.common.cache import MemcacheRegion
|
|
24
24
|
from rucio.common.exception import Duplicate, InvalidObject, RucioException
|
|
25
25
|
from rucio.db.sqla import models
|
|
26
26
|
from rucio.db.sqla.constants import KeyType
|
|
@@ -37,7 +37,7 @@ if TYPE_CHECKING:
|
|
|
37
37
|
scope: InternalScope
|
|
38
38
|
regexp: str
|
|
39
39
|
|
|
40
|
-
REGION =
|
|
40
|
+
REGION = MemcacheRegion(expiration_time=900)
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
@transactional_session
|
rucio/core/nongrid_trace.py
CHANGED
|
@@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, Any, Union, overload
|
|
|
20
20
|
|
|
21
21
|
import stomp
|
|
22
22
|
|
|
23
|
-
from rucio.common.config import config_get, config_get_int
|
|
23
|
+
from rucio.common.config import config_get, config_get_int, config_get_list
|
|
24
24
|
from rucio.common.logging import rucio_log_formatter
|
|
25
25
|
from rucio.core.monitor import MetricManager
|
|
26
26
|
|
|
@@ -58,7 +58,7 @@ ROTATING_LOGGER.addHandler(ROTATING_HANDLER)
|
|
|
58
58
|
|
|
59
59
|
BROKERS_ALIAS, BROKERS_RESOLVED = [], []
|
|
60
60
|
try:
|
|
61
|
-
BROKERS_ALIAS =
|
|
61
|
+
BROKERS_ALIAS = config_get_list('nongrid-trace', 'brokers')
|
|
62
62
|
except:
|
|
63
63
|
raise Exception('Could not load brokers from configuration')
|
|
64
64
|
|