rucio-clients 35.7.0__py3-none-any.whl

rucio/vcsversion.py

@@ -0,0 +1,11 @@
+
+'''
+This file is automatically generated; Do not edit it. :)
+'''
+VERSION_INFO = {
+    'final': True,
+    'version': '35.7.0',
+    'branch_nick': 'release-35-LTS',
+    'revision_id': 'c96dca5cf90ca93cb024f30f987ed8c48815a64d',
+    'revno': 13202
+}

rucio/version.py

@@ -0,0 +1,38 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from rucio.vcsversion import VERSION_INFO
+
+RUCIO_VERSION = [VERSION_INFO['version'], ]
+FINAL = VERSION_INFO['final']   # This becomes true at Release Candidate time
+
+
+def canonical_version_string() -> str:
+    """ Get the canonical string """
+    return '.'.join(filter(None, RUCIO_VERSION))
+
+
+def version_string() -> str:
+    """ Get the version string """
+    return canonical_version_string()
+
+
+def vcs_version_string() -> str:
+    """ Get the VCS version string """
+    return "%s:%s" % (VERSION_INFO['branch_nick'], VERSION_INFO['revision_id'])
+
+
+def version_string_with_vcs() -> str:
+    """ Get the version string with VCS """
+    return "%s-%s" % (canonical_version_string(), vcs_version_string())

rucio_clients-35.7.0.data/data/etc/rse-accounts.cfg.template

@@ -0,0 +1,25 @@
+{
+  "lxplus.cern.ch":{
+    "username":"USERNAME",
+    "password":"PASSWORD",
+    "host":"lxplus.cern.ch"
+  },
+  "swift.cern.ch": {
+    "access_key": "ACCESSKEY",
+    "secret_key": "PASSWORD",
+    "host_base": "swift.cern.ch",
+    "host_bucket": "swift.cern.ch",
+    "progress_meter": "False",
+    "skip_existing": "False"
+  },
+  "posix": {},
+  "DESY-ZN_SCRATCHDISK": {
+    "auth_type": "cert",
+    "timeout":300
+  },
+  "TAIWAN-LCG2_PPSSCRATCHDISK": {
+    "cert": ["/tmp/x509up","/tmp/x509up"],
+    "auth_type": "cert",
+    "timeout":300
+  }
+}

rucio_clients-35.7.0.data/data/etc/rucio.cfg.atlas.client.template

@@ -0,0 +1,42 @@
+# Copyright 2014-2021 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2014
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Nicolo Magini <nicolo.magini@cern.ch>, 2018
+# - Tomas Javurek <tomas.javurek@cern.ch>, 2019
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+
+[common]
+
+[client]
+rucio_host = https://voatlasrucio-server-prod.cern.ch:443
+auth_host = https://voatlasrucio-auth-prod.cern.ch:443
+client_x509_proxy = $X509_USER_PROXY
+request_retries = 3
+auth_type = x509_proxy
+
+[policy]
+permission = atlas
+schema = atlas
+lfn2pfn_algorithm_default = hash
+support = hn-atlas-dist-analysis-help@cern.ch
+support_rucio = https://github.com/rucio/rucio/issues/
+
+[upload]
+#transfer_timeout = 3600
+
+[download]
+#transfer_timeout = 3600

rucio_clients-35.7.0.data/data/etc/rucio.cfg.template

@@ -0,0 +1,257 @@
+# Copyright 2012-2022 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2017
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2021
+# - Luis Rodrigues <lfrodrigues@gmail.com>, 2013
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013
+# - Wen Guan <wen.guan@cern.ch>, 2014
+# - Fernando López <felopez@cern.ch>, 2015
+# - Martin Barisits <martin.barisits@cern.ch>, 2015-2021
+# - Vitjan Zavrtanik <vitjan.zavrtanik@cern.ch>, 2017
+# - Stefan Prenner <stefan.prenner@cern.ch>, 2018
+# - Nicolo Magini <nicolo.magini@cern.ch>, 2018
+# - Frank Berghaus <berghaus@cern.ch>, 2019
+# - Dilaksun Bavarajan <dilaksun.bavarajan@cern.ch>, 2019
+# - James Perry <j.perry@epcc.ed.ac.uk>, 2019
+# - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
+# - Jaroslav Guenther <jaroslav.guenther@cern.ch>, 2019
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
+# - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
+
+[common]
+logdir = /var/log/rucio
+loglevel = DEBUG
+logformat = %%(asctime)s\t%%(process)d\t%%(levelname)s\t%%(message)s
+mailtemplatedir=/opt/rucio/etc/mail_templates
+
+[client]
+rucio_host = https://rucio-server-prod.cern.ch:443
+auth_host = https://rucio-auth-prod.cern.ch:443
+auth_type = userpass
+#auth_type = gss
+#auth_type = x509_proxy
+#auth_type = ssh
+username = ddmlab
+password = secret
+ca_cert = /opt/rucio/etc/web/ca.crt
+#client_cert = /opt/rucio/etc/web/client.crt
+#client_key = /opt/rucio/etc/web/client.key
+#client_x509_proxy = $X509_USER_PROXY
+#ssh_private_key = $HOME/.ssh/id_rsa
+account = root
+request_retries = 3
+protocol_stat_retries = 6
+
+[upload]
+#transfer_timeout = 3600
+#preferred_impl = xrootd, rclone
+
+[download]
+#transfer_timeout = 3600
+#preferred_impl = xrootd, rclone
+
+[core]
+geoip_licence_key = LICENCEKEYGOESHERE  # Get a free licence key at https://www.maxmind.com/en/geolite2/signup
+default_mail_from = spamspamspam@cern.ch
+
+[database]
+default = sqlite:////tmp/rucio.db
+#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________)))
+#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
+#schema=atlas_rucio  # only for cern oracle
+#default = mysql+pymysql://rucio:rucio@localhost/rucio
+#default = postgresql://rucio:rucio@localhost/rucio
+pool_recycle=3600
+echo=0
+pool_reset_on_return=rollback
+# Uncomment the following line to disable database connection pooling.
+#poolclass = nullpool
+
+[bootstrap]
+# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest()
+userpass_identity = ddmlab
+userpass_pwd = 2ccee6f6dd1bc2269cddd7cd5e47578e98e430539807c36df23fab7dd13e7583
+userpass_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt
+x509_identity = emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH
+x509_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB cern account
+gss_identity = ddmlab@CERN.CH
+gss_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB ssh key
+ssh_identity = ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydnVy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab
+ssh_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB SAML Name ID
+saml_id = ddmlab
+saml_email = ph-adp-ddm-lab@cern.ch
+
+[monitor]
+carbon_server = rucio-graphite-int.cern.ch
+carbon_port = 8125
+user_scope = your_username
+enable_metrics = False
+metrics_port = 8080
+
+[conveyor]
+scheme = srm,gsiftp,root,http,https
+transfertool = fts3
+ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
+cacert = /opt/rucio/etc/web/ca.crt
+usercert = /opt/rucio/tools/x509up
+
+[messaging-fts3]
+port = 61123
+ssl_key_file = /home/mario/.ssh/hostkey.pem
+ssl_cert_file = /home/mario/.ssh/hostcert.pem
+destination = /topic/transfer.fts_monitoring_queue_state
+brokers = dashb-test-mb.cern.ch
+voname = atlas
+
+[messaging-hermes]
+username = ____
+password = ____
+port = 61023
+nonssl_port = 61013
+use_ssl = True
+ssl_key_file = /etc/grid-security/hostkey.pem
+ssl_cert_file = /etc/grid-security/hostcert.pem
+destination = /topic/rucio.events
+brokers = localhost
+voname = atlas
+email_from = Rucio <spamspamspam@cern.ch>
+email_test = spamspamspam@cern.ch
+
+[transmogrifier]
+maxdids = 100000
+
+[accounts]
+# These are accounts that can write into scopes owned by another account
+special_accounts = panda, tier0
+
+[trace]
+tracedir = /var/log/rucio/trace
+brokers=localhost
+port=61013
+username = _________
+password = _________
+topic = /topic/rucio.tracer
+trace_host = https://rucio-server-prod.cern.ch:443
+
+[tracer-kronos]
+brokers=localhost
+port=61013
+ssl_key_file = /etc/grid-security/hostkey.pem
+ssl_cert_file = /etc/grid-security/hostcert.pem
+queue = /queue/Consumer.kronos.rucio.tracer
+prefetch_size = 10
+chunksize = 10
+subscription_id = rucio-tracer-listener
+use_ssl = False
+reconnect_attempts = 100
+excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch
+username = _________
+password = _________
+dataset_wait = 60
+
+[injector]
+file = /opt/rucio/tools/test.file.1000
+bytes = 1000
+md5 = fd21ce524a9e45060fd3f62c4ef6a386
+adler32 = 52590737
+#file = /opt/rucio/tools/test.file.1G
+#bytes = 1000000000
+#md5sum = 1a3031f43cc0c0cf4f8341920a9faf52
+#adler32 = 5ff2d417
+
+[alembic]
+cfg = /opt/rucio/etc/alembic.ini
+
+[messaging-cache]
+port = 61023
+ssl_key_file = /etc/grid-security/hostkey.pem
+ssl_cert_file = /etc/grid-security/hostcert.pem
+destination = /topic/rucio.fax
+brokers = localhost
+voname = atlas
+account = cache_mb
+
+[test]
+cacert = /opt/rucio/etc/web/ca.crt
+#cacert = /etc/pki/tls/certs/CERN-bundle.pem
+usercert = /opt/rucio/etc/web/usercert.pem
+#usercert = /home/mario/.ssh/usercert_with_key.pem
+
+[nagios]
+proxy = /opt/rucio/etc/ddmadmin.proxy.nagios
+rfcproxy = /opt/rucio/etc/ddmusr01.rfc.proxy
+fts_servers = https://fts3-pilot.cern.ch:8446, https://fts3-devel.cern.ch:8446, https://fts3.cern.ch:8446, https://lcgfts3.gridpp.rl.ac.uk:8446, https://fts3-test.gridpp.rl.ac.uk:8446, https://fts.usatlas.bnl.gov:8446
+
+[auditor]
+cache = /opt/rucio/auditor-cache
+results = /opt/rucio/auditor-results
+
+[c3po]
+placement_algorithm = t2_free_space
+elastic_url = http://aianalytics01.cern.ch:9200
+redis_host = localhost
+redis_port = 6379
+
+[c3po-popularity]
+elastic_url = http://rucio-logger-prod-01.cern.ch:9200
+
+[c3po-site-mapper]
+panda_url = http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json
+ddm_url = http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json
+
+[c3po-workload]
+panda_url = http://bigpanda.cern.ch/jobs/?category=analysis&jobstatus=running
+window = 604800
+
+[policy]
+package = ATLASRucioPolicy
+permission = atlas
+schema = atlas
+lfn2pfn_algorithm_default = hash
+support = hn-atlas-dist-analysis-help@cern.ch
+support_rucio = https://github.com/rucio/rucio/issues/
+
+[webui]
+usercert = /opt/rucio/etc/usercert_with_key.pem
+
+[credentials]
+gcs = /opt/rucio/etc/google-cloud-storage-test.json
+signature_lifetime = 3600
+
+[saml]
+config_path = /opt/rucio/lib/rucio/web/ui/common/saml/
+
+[oidc]
+idpsecrets = /opt/rucio/etc/idpsecrets.json
+admin_issuer = wlcg
+
+[api]
+#endpoints = accountlimits, accounts, config, credentials, dids, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions
+
+[transfers]
+srm_https_compatibility = False

rucio_clients-35.7.0.data/data/requirements.client.txt

@@ -0,0 +1,15 @@
+# All dependencies needed to run rucio client should be defined here
+requests>=2.32.2                                            # Python HTTP for Humans.
+urllib3>=1.26.18                                            # HTTP library with thread-safe connection pooling, file post, etc.
+dogpile-cache>=1.2.2                                        # Caching API plugins (1.1.2 is the first version to support pymemcache)
+tabulate>=0.9.0                                             # Pretty-print tabular data
+jsonschema>=4.20.0                                          # For JSON schema validation (Policy modules)
+
+# All dependencies needed in extras for rucio client should be defined here
+paramiko>=3.4.1                                             # ssh_extras; SSH2 protocol library (also needed in the server)
+kerberos>=1.3.1                                             # kerberos_extras for client and server
+pykerberos>=1.2.4                                           # kerberos_extras for client and server
+requests-kerberos>=0.14.0                                   # kerberos_extras for client and server
+python-swiftclient>=4.4.0                                   # swift_extras
+argcomplete>=3.1.6                                          # argcomplete_extras; Bash tab completion for argparse
+python-magic>=0.4.27                                        # dumper_extras; File type identification using libmagic

rucio_clients-35.7.0.data/data/rucio_client/merge_rucio_configs.py

@@ -0,0 +1,144 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+import configparser
+
+# Multi-word sections used in kubernetes are slightly different from what rucio expects.
+# Usually, it's just a .replace('-', '_'), but not for hermes2, which doesn't follow any convention.
+multi_word_sections = {
+    'messaging_fts3': 'messaging-fts3',
+    'messaging_cache': 'messaging-cache',
+    'messaging_hermes': 'messaging-hermes',
+    'messaging_hermes2': 'hermes',
+    'nongrid_trace': 'nongrid-trace',
+    'tracer_kronos': 'tracer-kronos',
+}
+
+
+def load_flat_config(flat_config):
+    """
+    takes a dict of the form: {"section_option": "value"}
+    and converts to {"section": {"option": "value"}
+    """
+    config_dict = {}
+    for flat_key, config_value in flat_config.items():
+        section = option = None
+        # Try parsing a multi-word section
+        for mw_key in multi_word_sections:
+            if flat_key.startswith(mw_key + '_'):
+                section = mw_key
+                option = flat_key[len(mw_key) + 1:]
+
+        # It didn't match any known multi-word section, assume it's a single word
+        if not section:
+            section, option = flat_key.split('_', maxsplit=1)
+
+        config_dict.setdefault(section, {})[option] = config_value
+    return config_dict
+
+
+def fix_multi_word_sections(config_dict):
+    return {multi_word_sections.get(section, section): config_for_section for section, config_for_section in config_dict.items()}
+
+
+def config_len(config_dict):
+    return sum(len(option) for _, option in config_dict.items())
+
+
+def merge_configs(source_file_paths, dest_file_path, use_env=True, logger=logging.log):
+    """
+    Merge multiple configuration sources into one rucio.cfg.
+    On conflicting values, relies on the default python's ConfigParser behavior: the value from last source wins.
+    Sources can be .ini, .yaml, or .json files. Json is supported as a compromise solution for easier integration
+    with kubernetes (because both python and helm natively support it).
+    If use_env=True, env variables starting with RUCIO_CFG_ are also merged as the last (highest priority) source.
+    """
+
+    parser = configparser.ConfigParser()
+    for path in source_file_paths:
+        path = Path(path)
+
+        if not path.exists():
+            logger(logging.WARNING, "Skipping {}: path doesn't exist".format(path))
+            continue
+
+        if path.is_dir():
+            file_paths = sorted(p for p in path.iterdir() if not p.name.startswith(".") and p.is_file())
+        else:
+            file_paths = [path]
+
+        for file_path in file_paths:
+            try:
+                if file_path.suffix == '.json':
+                    with open(file_path, 'r') as f:
+                        file_config = fix_multi_word_sections(json.load(f))
+                        parser.read_dict(file_config)
+                elif yaml and file_path.suffix in ['.yaml', '.yml']:
+                    with open(file_path, 'r') as f:
+                        file_config = fix_multi_word_sections(yaml.safe_load(f))
+                        parser.read_dict(file_config)
+                elif path.is_file() or file_path.suffix in ['.ini', '.cfg', '.config']:
+                    local_parser = configparser.ConfigParser()
+                    local_parser.read(file_path)
+                    file_config = {section: {option: value for option, value in section_proxy.items()} for section, section_proxy in local_parser.items()}
+                else:
+                    logger(logging.WARNING, "Skipping file {} due to wrong extension".format(file_path))
+                    continue
+
+                parser.read_dict(file_config)
+                logger(logging.INFO, "Merged {} configuration values from {}".format(config_len(file_config), file_path))
+            except Exception as error:
+                logger(logging.WARNING, "Skipping file {} due to error: {}".format(file_path, error))
+
+    if use_env:
+        # env variables use the following format: "RUCIO_CFG_{section.substitute('-','_').upper}_{option.substitute('-', '_').upper}"
+        env_config = {}
+        for env_key, env_value in os.environ.items():
+            rucio_cfg_prefix = 'RUCIO_CFG_'
+            if not env_key.startswith(rucio_cfg_prefix):
+                continue
+            env_key = env_key[len(rucio_cfg_prefix):].lower()  # convert "RUCIO_CFG_WHATEVER" to "whatever"
+            env_config[env_key] = env_value
+
+        env_config = fix_multi_word_sections(load_flat_config(env_config))
+        parser.read_dict(env_config)
+        logger(logging.INFO, "Merged {} configuration values from ENV".format(config_len(env_config)))
+
+    if dest_file_path:
+        logger(logging.INFO, "Writing {}".format(dest_file_path))
+        with open(dest_file_path, 'w') as dest_file:
+            parser.write(dest_file)
+    else:
+        parser.write(sys.stdout)
+
+
+logging.getLogger().setLevel(logging.INFO)
+parser = argparse.ArgumentParser(description="Merge multiple rucio configuration sources into one rucio.cfg")
+parser.add_argument("--use-env", action="store_true", default=False, help='Also source config from RUCIO_CFG_* env variables')
+parser.add_argument('-s', '--source', type=str, nargs='*', help='Source config file paths (in .json, .yaml or .ini format)')
+parser.add_argument('-d', '--destination', default=None, help='Destination file path')
+args = parser.parse_args()
+
+merge_configs(args.source or [], args.destination, use_env=args.use_env)