rucio 32.8.6__py3-none-any.whl

rucio-32.8.6.data/data/rucio/etc/rucio_multi_vo.cfg.template

@@ -0,0 +1,234 @@
+# Copyright 2020-2022 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
+
+[common]
+logdir = /var/log/rucio
+loglevel = DEBUG
+mailtemplatedir=/opt/rucio/etc/mail_templates
+multi_vo = True
+
+[client]
+rucio_host = https://rucio-server-prod.cern.ch:443
+auth_host = https://rucio-auth-prod.cern.ch:443
+auth_type = userpass
+#auth_type = gss
+#auth_type = x509_proxy
+#auth_type = ssh
+username = ddmlab
+password = secret
+ca_cert = /opt/rucio/etc/web/ca.crt
+#client_cert = /opt/rucio/etc/web/client.crt
+#client_key = /opt/rucio/etc/web/client.key
+#client_x509_proxy = $X509_USER_PROXY
+#ssh_private_key = $HOME/.ssh/id_rsa
+account = root
+vo = tst
+request_retries = 3
+protocol_stat_retries = 6
+
+[upload]
+#transfer_timeout = 3600
+
+[download]
+#transfer_timeout = 3600
+
+[core]
+geoip_licence_key = LICENCEKEYGOESHERE  # Get a free licence key at https://www.maxmind.com/en/geolite2/signup
+default_mail_from = spamspamspam@cern.ch
+
+[database]
+default = sqlite:////tmp/rucio.db
+#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________)))
+#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
+#schema=atlas_rucio  # only for cern oracle
+#default = mysql+pymysql://rucio:rucio@localhost/rucio
+#default = postgresql://rucio:rucio@localhost/rucio
+pool_recycle=3600
+echo=0
+pool_reset_on_return=rollback
+
+[bootstrap]
+# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest()
+userpass_identity = ddmlab
+userpass_pwd = 2ccee6f6dd1bc2269cddd7cd5e47578e98e430539807c36df23fab7dd13e7583
+userpass_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt
+x509_identity = emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH
+x509_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB cern account
+gss_identity = ddmlab@CERN.CH
+gss_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB ssh key
+ssh_identity = ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydnVy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab
+ssh_email = ph-adp-ddm-lab@cern.ch
+
+# Default DDMLAB SAML Name ID
+saml_id = ddmlab
+saml_email = ph-adp-ddm-lab@cern.ch
+
+[monitor]
+carbon_server = rucio-graphite-int.cern.ch
+carbon_port = 8125
+user_scope = your_username
+enable_metrics = False
+metrics_port = 8080
+
+[conveyor]
+scheme = srm,gsiftp,root,http,https
+transfertool = fts3
+ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
+cacert = /opt/rucio/etc/web/ca.crt
+usercert = /opt/rucio/tools/x509up
+
+[messaging-fts3]
+port = 61123
+ssl_key_file = /home/mario/.ssh/hostkey.pem
+ssl_cert_file = /home/mario/.ssh/hostcert.pem
+destination = /topic/transfer.fts_monitoring_queue_state
+brokers = dashb-test-mb.cern.ch
+voname = atlas
+
+[messaging-hermes]
+username = ____
+password = ____
+port = 61023
+nonssl_port = 61013
+use_ssl = True
+ssl_key_file = /etc/grid-security/hostkey.pem
+ssl_cert_file = /etc/grid-security/hostcert.pem
+destination = /topic/rucio.events
+brokers = localhost
+voname = atlas
+email_from = Rucio <spamspamspam@cern.ch>
+email_test = spamspamspam@cern.ch
+
+[transmogrifier]
+maxdids = 100000
+
+[accounts]
+# These are accounts that can write into scopes owned by another account
+special_accounts = panda, tier0
+
+[trace]
+tracedir = /var/log/rucio/trace
+brokers=localhost
+port=61013
+username = _________
+password = _________
+topic = /topic/rucio.tracer
+
+[tracer-kronos]
+brokers=localhost
+port=61013
+ssl_key_file = /etc/grid-security/hostkey.pem
+ssl_cert_file = /etc/grid-security/hostcert.pem
+queue = /queue/Consumer.kronos.rucio.tracer
+prefetch_size = 10
+chunksize = 10
+subscription_id = rucio-tracer-listener
+use_ssl = False
+reconnect_attempts = 100
+excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch
+username = _________
+password = _________
+dataset_wait = 60
+
+[injector]
+file = /opt/rucio/tools/test.file.1000
+bytes = 1000
+md5 = fd21ce524a9e45060fd3f62c4ef6a386
+adler32 = 52590737
+#file = /opt/rucio/tools/test.file.1G
+#bytes = 1000000000
+#md5sum = 1a3031f43cc0c0cf4f8341920a9faf52
+#adler32 = 5ff2d417
+
+[alembic]
+cfg = /opt/rucio/etc/alembic.ini
+
+[messaging-cache]
+port = 61023
+ssl_key_file = /etc/grid-security/hostkey.pem
+ssl_cert_file = /etc/grid-security/hostcert.pem
+destination = /topic/rucio.fax
+brokers = localhost
+voname = atlas
+account = cache_mb
+
+[test]
+cacert = /opt/rucio/etc/web/ca.crt
+#cacert = /etc/pki/tls/certs/CERN-bundle.pem
+usercert = /opt/rucio/etc/web/usercert.pem
+#usercert = /home/mario/.ssh/usercert_with_key.pem
+
+[nagios]
+proxy = /opt/rucio/etc/ddmadmin.proxy.nagios
+rfcproxy = /opt/rucio/etc/ddmusr01.rfc.proxy
+fts_servers = https://fts3-pilot.cern.ch:8446, https://fts3-devel.cern.ch:8446, https://fts3.cern.ch:8446, https://lcgfts3.gridpp.rl.ac.uk:8446, https://fts3-test.gridpp.rl.ac.uk:8446, https://fts.usatlas.bnl.gov:8446
+
+[auditor]
+cache = /opt/rucio/auditor-cache
+results = /opt/rucio/auditor-results
+
+[c3po]
+placement_algorithm = t2_free_space
+elastic_url = http://aianalytics01.cern.ch:9200
+redis_host = localhost
+redis_port = 6379
+
+[c3po-popularity]
+elastic_url = http://rucio-logger-prod-01.cern.ch:9200
+
+[c3po-site-mapper]
+panda_url = http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json
+ddm_url = http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json
+
+[c3po-workload]
+panda_url = http://bigpanda.cern.ch/jobs/?category=analysis&jobstatus=running
+window = 604800
+
+[policy]
+#package = GenericMultiVORucioPolicy #doesn't exist
+permission = generic_multi_vo
+schema = generic_multi_vo
+lfn2pfn_algorithm_default = hash
+support = hn-atlas-dist-analysis-help@cern.ch
+support_rucio = https://github.com/rucio/rucio/issues/
+
+[webui]
+usercert = /opt/rucio/etc/usercert_with_key.pem
+
+[credentials]
+gcs = /opt/rucio/etc/google-cloud-storage-test.json
+signature_lifetime = 3600
+
+[saml]
+config_path = /opt/rucio/lib/rucio/web/ui/common/saml/
+
+[oidc]
+idpsecrets = /opt/rucio/etc/idpsecrets.json
+admin_issuer = wlcg
+
+[api]
+endpoints = accountlimits, accounts, config, credentials, dids, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, traces, vos

rucio-32.8.6.data/data/rucio/requirements.txt

@@ -0,0 +1,55 @@
+# All dependencies needed to run rucio client (and server/daemons) should be defined here
+requests>=2.25.1,<=2.31.0                                   # Python HTTP for Humans.
+urllib3~=1.26.16                                            # HTTP library with thread-safe connection pooling, file post, etc.
+dogpile.cache~=1.2.2                                        # Caching API plugins (1.1.2 is the first version to support pymemcache)
+tabulate~=0.9.0                                             # Pretty-print tabular data
+jsonschema~=4.18.4                                          # For JSON schema validation (Policy modules)
+
+# All dependencies needed in extras for rucio client (and server/daemons) should be defined here
+paramiko~=3.4.0                                             # ssh_extras; SSH2 protocol library (also needed in the server)
+kerberos~=1.3.1                                             # kerberos_extras for client and server
+pykerberos~=1.2.4                                           # kerberos_extras for client and server
+requests-kerberos>=0.14.0                                   # kerberos_extras for client and server
+python-swiftclient~=4.3.0                                   # swift_extras
+argcomplete~=3.1.1                                          # argcomplete_extras; Bash tab completion for argparse
+python-magic~=0.4.27                                        # dumper_extras; File type identification using libmagic
+
+# All dependencies needed to run rucio server/daemons should be defined here
+SQLAlchemy==2.0.19                                          # DB backend
+alembic~=1.11.1                                             # Lightweight database migration tool for SQLAlchemy
+pymemcache==4.0.0                                           # A comprehensive, fast, pure-Python memcached client (Used by Dogpile)
+python-dateutil==2.8.2                                      # Extensions to the standard datetime module
+stomp.py==8.1.0                                             # ActiveMQ Messaging Protocol
+statsd==4.0.1                                               # Needed to log into graphite with more than 1 Hz
+geoip2==4.7.0                                               # GeoIP2 API (for IPv6 support)
+google-auth==2.22.0                                         # Google authentication library for Python
+redis==4.6.0                                                # Python client for Redis key-value store
+Flask==2.3.2                                                # Python web framework
+oic==1.6.1                                                  # for authentication via OpenID Connect protocol
+prometheus_client==0.17.1                                   # Python client for the Prometheus monitoring system
+boto3==1.28.5                                               # S3 boto protocol (new version)
+
+# All dependencies needed in extras for rucio server/daemons should be defined here
+cx_oracle==8.3.0                                            # oracle_extras
+psycopg2-binary==2.9.6                                      # postgresql_extras
+PyMySQL==1.1.0                                              # mysql_extras
+PyYAML==6.0.1                                               # globus_extras and used for reading test configuration files
+globus-sdk==3.24.0                                          # globus_extras
+python3-saml==1.15.0                                        # saml_extras
+pymongo==4.4.1                                              # pymongo (metadata plugin)
+
+# All dependencies needed to develop/test rucio should be defined here
+pytest==7.4.0
+pytest-xdist~=3.3.1                                         # Used for parallel testing
+pyflakes==3.0.1                                             # Passive checker of Python programs
+flake8==6.0.0                                               # Wrapper around PyFlakes&pep8; python_version < '3.9'
+pycodestyle==2.10.0                                         # New package replacing pep8; python_version < '3.9'
+pylint==3.0.0a7
+astroid==3.0.0a9
+virtualenv==20.24.0                                         # Virtual Python Environment builder
+xmltodict==0.13.0                                           # Makes working with XML feel like you are working with JSON
+pytz==2023.3                                                # World timezone definitions, modern and historical
+pydoc-markdown~=4.8.2                                       # Used for generating Markdown documentation for docusaurus
+sh~=2.0.4                                                   # Convenience library for running subprocesses in Python
+apispec==6.3.0                                              # Generate OpenApi definition out of pydoc comments
+apispec-webframeworks                                       # Integration of apispec in Flask

rucio-32.8.6.data/data/rucio/tools/bootstrap.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run this once to set up the database.
+#   PYTHONPATH=/opt/rucio/.venv/lib/python2.7/site-packages/rucio python tools/bootstrap.py
+#
+# Verify for default SQLite:
+#   for i in `sqlite3 /tmp/rucio.db ".tables"`; do echo $i:; sqlite3 /tmp/rucio.db "select * from $i"; echo; done
+
+import sys
+import os.path
+base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(base_path)
+os.chdir(base_path)
+
+from rucio.db.sqla.util import (build_database, create_base_vo, create_root_account)  # noqa: E402
+
+if __name__ == '__main__':
+    build_database()
+    create_base_vo()
+    create_root_account()

rucio-32.8.6.data/data/rucio/tools/merge_rucio_configs.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+from pathlib import Path
+
+import os
+import json
+import sys
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+import configparser
+
+
+# Multi-word sections used in kubernetes are slightly different from what rucio expects.
+# Usually, it's just a .replace('-', '_'), but not for hermes2, which doesn't follow any convention.
+multi_word_sections = {
+    'messaging_fts3': 'messaging-fts3',
+    'messaging_cache': 'messaging-cache',
+    'messaging_hermes': 'messaging-hermes',
+    'messaging_hermes2': 'hermes',
+    'nongrid_trace': 'nongrid-trace',
+    'tracer_kronos': 'tracer-kronos',
+}
+
+
+def load_flat_config(flat_config):
+    """
+    takes a dict of the form: {"section_option": "value"}
+    and converts to {"section": {"option": "value"}
+    """
+    config_dict = {}
+    for flat_key, config_value in flat_config.items():
+        section = option = None
+        # Try parsing a multi-word section
+        for mw_key in multi_word_sections:
+            if flat_key.startswith(mw_key + '_'):
+                section = mw_key
+                option = flat_key[len(mw_key) + 1:]
+
+        # It didn't match any known multi-word section, assume it's a single word
+        if not section:
+            section, option = flat_key.split('_', maxsplit=1)
+
+        config_dict.setdefault(section, {})[option] = config_value
+    return config_dict
+
+
+def fix_multi_word_sections(config_dict):
+    return {multi_word_sections.get(section, section): config_for_section for section, config_for_section in config_dict.items()}
+
+
+def config_len(config_dict):
+    return sum(len(option) for _, option in config_dict.items())
+
+
+def merge_configs(source_file_paths, dest_file_path, use_env=True, logger=logging.log):
+    """
+    Merge multiple configuration sources into one rucio.cfg.
+    On conflicting values, relies on the default python's ConfigParser behavior: the value from last source wins.
+    Sources can be .ini, .yaml, or .json files. Json is supported as a compromise solution for easier integration
+    with kubernetes (because both python and helm natively support it).
+    If use_env=True, env variables starting with RUCIO_CFG_ are also merged as the last (highest priority) source.
+    """
+
+    parser = configparser.ConfigParser()
+    for path in source_file_paths:
+        path = Path(path)
+
+        if not path.exists():
+            logger(logging.WARNING, "Skipping {}: path doesn't exist".format(path))
+            continue
+
+        if path.is_dir():
+            file_paths = sorted(p for p in path.iterdir() if not p.name.startswith(".") and p.is_file())
+        else:
+            file_paths = [path]
+
+        for file_path in file_paths:
+            try:
+                if file_path.suffix == '.json':
+                    with open(file_path, 'r') as f:
+                        file_config = fix_multi_word_sections(json.load(f))
+                        parser.read_dict(file_config)
+                elif yaml and file_path.suffix in ['.yaml', '.yml']:
+                    with open(file_path, 'r') as f:
+                        file_config = fix_multi_word_sections(yaml.safe_load(f))
+                        parser.read_dict(file_config)
+                elif path.is_file() or file_path.suffix in ['.ini', '.cfg', '.config']:
+                    local_parser = configparser.ConfigParser()
+                    local_parser.read(file_path)
+                    file_config = {section: {option: value for option, value in section_proxy.items()} for section, section_proxy in local_parser.items()}
+                else:
+                    logger(logging.WARNING, "Skipping file {} due to wrong extension".format(file_path))
+                    continue
+
+                parser.read_dict(file_config)
+                logger(logging.INFO, "Merged {} configuration values from {}".format(config_len(file_config), file_path))
+            except Exception as error:
+                logger(logging.WARNING, "Skipping file {} due to error: {}".format(file_path, error))
+
+    if use_env:
+        # env variables use the following format: "RUCIO_CFG_{section.substitute('-','_').upper}_{option.substitute('-', '_').upper}"
+        env_config = {}
+        for env_key, env_value in os.environ.items():
+            rucio_cfg_prefix = 'RUCIO_CFG_'
+            if not env_key.startswith(rucio_cfg_prefix):
+                continue
+            env_key = env_key[len(rucio_cfg_prefix):].lower()  # convert "RUCIO_CFG_WHATEVER" to "whatever"
+            env_config[env_key] = env_value
+
+        env_config = fix_multi_word_sections(load_flat_config(env_config))
+        parser.read_dict(env_config)
+        logger(logging.INFO, "Merged {} configuration values from ENV".format(config_len(env_config)))
+
+    if dest_file_path:
+        logger(logging.INFO, "Writing {}".format(dest_file_path))
+        with open(dest_file_path, 'w') as dest_file:
+            parser.write(dest_file)
+    else:
+        parser.write(sys.stdout)
+
+
+logging.getLogger().setLevel(logging.INFO)
+parser = argparse.ArgumentParser(description="Merge multiple rucio configuration sources into one rucio.cfg")
+parser.add_argument("--use-env", action="store_true", default=False, help='Also source config from RUCIO_CFG_* env variables')
+parser.add_argument('-s', '--source', type=str, nargs='*', help='Source config file paths (in .json, .yaml or .ini format)')
+parser.add_argument('-d', '--destination', default=None, help='Destination file path')
+args = parser.parse_args()
+
+merge_configs(args.source or [], args.destination, use_env=args.use_env)

rucio-32.8.6.data/data/rucio/tools/reset_database.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os.path
+base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(base_path)
+os.chdir(base_path)
+
+from argparse import ArgumentParser  # noqa: E402
+
+from rucio.db.sqla.util import build_database, destroy_database, drop_everything, create_root_account, create_base_vo  # noqa: E402
+
+if __name__ == '__main__':
+
+    parser = ArgumentParser()
+    parser.add_argument('-d', '--drop-everything', action="store_true", default=False, help='Drop all tables and constraints')
+    args = parser.parse_args()
+
+    if args.drop_everything:
+        drop_everything()
+    else:
+        destroy_database()
+
+    build_database()
+    create_base_vo()
+    create_root_account()