rucio 34.6.0__tar.gz → 35.0.0rc2__tar.gz

{rucio-34.6.0 → rucio-35.0.0rc2}/AUTHORS.rst

@@ -66,6 +66,7 @@ Individual contributors to the source code
 - Jensen Zhang <hack@jensen-zhang.site>, 2022
 - Aksel Lunde Aase <aksel.lunde.aase@gmail.com>, 2022
 - Elena Gazzarrini <gazzarrini.elena@gmail.com>, 2022-2023
+- Maximilian Linhoff, <maximilian.linhoff@tu-dortmund.de>, 2024
 
 Organisations employing contributors
 ------------------------------------
@@ -93,3 +94,4 @@ Organisations employing contributors
 - Imperial College London (UK)
 - University of Chicago (USA)
 - Purdue University (USA)
+- TU Dortmund University (Germany)

{rucio-34.6.0 → rucio-35.0.0rc2}/MANIFEST.in

@@ -7,7 +7,7 @@ include setuputil.py
 include setup.py
 include setup.cfg
 include pyproject.toml
-include requirements.txt
+include requirements/requirements.server.txt
 include lib/rucio/db/migrate_repo/script.py.mako
 include etc/rse-accounts.cfg.template
 include etc/rucio.cfg.atlas.client.template

rucio-35.0.0rc2/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.1
+Name: rucio
+Version: 35.0.0rc2
+Summary: Rucio Package
+Home-page: https://rucio.cern.ch/
+Author: Rucio
+Author-email: rucio-dev@cern.ch
+License: Apache License, Version 2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Environment :: No Input/Output (Daemon)
+Requires-Python: >=3.9, <4
+License-File: LICENSE
+License-File: AUTHORS.rst
+Requires-Dist: alembic==1.13.2
+Requires-Dist: argcomplete==3.4.0
+Requires-Dist: boto3==1.34.142
+Requires-Dist: dogpile-cache==1.2.2
+Requires-Dist: flask==3.0.3
+Requires-Dist: geoip2==4.8.0
+Requires-Dist: google-auth==2.32.0
+Requires-Dist: jsonschema==4.23.0
+Requires-Dist: oic==1.7.0
+Requires-Dist: paramiko==3.4.0
+Requires-Dist: prometheus-client==0.20.0
+Requires-Dist: pymemcache==4.0.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-magic==0.4.27
+Requires-Dist: redis==5.0.7
+Requires-Dist: requests==2.32.3
+Requires-Dist: sqlalchemy==2.0.31
+Requires-Dist: statsd==4.0.1
+Requires-Dist: stomp-py==8.1.2
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: urllib3==1.26.19
+Provides-Extra: oracle
+Requires-Dist: cx-oracle==8.3.0; extra == "oracle"
+Provides-Extra: mongo
+Requires-Dist: pymongo==4.8.0; extra == "mongo"
+Provides-Extra: postgresql
+Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"
+Provides-Extra: mysql
+Requires-Dist: pymysql==1.1.1; extra == "mysql"
+Provides-Extra: kerberos
+Requires-Dist: kerberos==1.3.1; extra == "kerberos"
+Requires-Dist: pykerberos==1.2.4; extra == "kerberos"
+Requires-Dist: requests-kerberos==0.15.0; extra == "kerberos"
+Provides-Extra: globus
+Requires-Dist: globus-sdk==3.41.0; extra == "globus"
+Requires-Dist: pyyaml==6.0.1; extra == "globus"
+Provides-Extra: saml
+Requires-Dist: python3-saml==1.16.0; extra == "saml"
+Provides-Extra: dev
+Requires-Dist: pyyaml==6.0.1; extra == "dev"

rucio-35.0.0rc2/README.md

@@ -0,0 +1,27 @@
+# Rucio - Scientific Data Management
+
+Rucio is a software framework that provides functionality to organize, manage, and access large volumes of scientific data using customisable policies.
+The data can be spread across globally distributed locations and across heterogeneous data centers, uniting different storage and network technologies as a single federated entity.
+Rucio offers advanced features such as distributed data recovery or adaptive replication, and is highly scalable, modular, and extensible.
+Rucio has been originally developed to meet the requirements of the high-energy physics experiment ATLAS, and is continuously extended to support LHC experiments and other diverse scientific communities.
+
+## Documentation
+
+General information, API/REST description and guides can be found in our [documentation](https://rucio.cern.ch/documentation) or on our [webpage](https://rucio.cern.ch).
+
+## Try it out
+
+We provide a [dockerized environment](https://github.com/rucio/rucio/tree/master/etc/docker/dev) which serves both as a demo environment and a development environment.
+It includes all the necessary preconfigured components for multiple storage and transfers developments.
+
+## Developers
+
+For information on how to contribute to Rucio, please refer and follow our [CONTRIBUTING](https://rucio.cern.ch/documentation/contributing) guidelines. We strongly recommend to use the [dockerized environment](https://github.com/rucio/rucio/tree/master/etc/docker/dev) for development.
+
+## Operators
+
+To learn how to deploy and configure Rucio, consult the [documentation](https://rucio.cern.ch/documentation) available online.
+
+## Getting Support
+
+If you are looking for support, please contact us via one of our [official channels](https://rucio.cern.ch/documentation/contact_us/).

{rucio-34.6.0 → rucio-35.0.0rc2}/bin/rucio

@@ -30,6 +30,7 @@ from configparser import NoOptionError, NoSectionError
 from copy import deepcopy
 from datetime import datetime
 from functools import wraps
+from typing import Optional
 
 from tabulate import tabulate
 
@@ -1079,6 +1080,24 @@ def download(args):
             logger.debug(args.dids)
         item_defaults['pfn'] = args.pfn
         item_defaults['did'] = did_str
+        if args.rses is None:
+            logger.warning("No RSE was given, selecting one.")
+            client = get_client(args)
+            replicas = client.list_replicas(
+                [{"scope": did_str.split(':')[0], "name": did_str.split(':')[-1]}],
+                schemes=args.protocol,
+                ignore_availability=False,
+                client_location=detect_client_location(),
+                resolve_archives=not args.no_resolve_archives
+            )
+
+            download_rse = _get_rse_for_pfn(replicas, args.pfn)
+            if download_rse is None:
+                logger.error("Could not find RSE for pfn %s", args.pfn)
+                return FAILURE
+            else:
+                item_defaults['rse'] = download_rse
+
         result = download_client.download_pfns([item_defaults], 1, trace_pattern, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
 
     if not result:
@@ -1120,6 +1139,21 @@ def download(args):
     return SUCCESS
 
 
+def _get_rse_for_pfn(replicas, pfn) -> Optional[str]:
+    # Check each rse in the replica list for the pfn. If no pfn is found, returns None.
+    # If it is found, stop the generator and return the item.
+    for replica in replicas:
+        try:
+            download_rse = next(
+                rse for rse in replica['rses']
+                if pfn in replica['rses'][rse]
+            )
+        except StopIteration:
+            continue
+        else:
+            return download_rse
+
+
 @exception_handler
 def get_metadata(args):
     """

{rucio-34.6.0 → rucio-35.0.0rc2}/bin/rucio-admin

@@ -54,7 +54,7 @@ from rucio.common.exception import (
     ScopeNotFound,
 )
 from rucio.common.extra import import_extras
-from rucio.common.utils import StoreAndDeprecateWarningAction, chunks, clean_surls, construct_surl, extract_scope, get_bytes_value_from_string, parse_response, render_json, sizefmt
+from rucio.common.utils import StoreAndDeprecateWarningAction, chunks, clean_pfns, construct_non_deterministic_pfn, extract_scope, get_bytes_value_from_string, parse_response, render_json, sizefmt
 from rucio.rse import rsemanager as rsemgr
 
 EXTRA_MODULES = import_extras(['argcomplete'])
@@ -1148,21 +1148,21 @@ def declare_bad_file_replicas(args):
             list_bad_pfns = []
             cnt += 1
             previous_pattern = None
-            for surl in clean_surls(chunk):
+            for pfn in clean_pfns(chunk):
                 unknown = True
                 if previous_pattern:
-                    if previous_pattern in surl:
-                        list_bad_pfns.append(surl)
+                    if previous_pattern in pfn:
+                        list_bad_pfns.append(pfn)
                         unknown = False
                         continue
                 for pattern in prot_dict:
-                    if pattern in surl:
+                    if pattern in pfn:
                         previous_pattern = prot_dict[pattern]
-                        list_bad_pfns.append(surl)
+                        list_bad_pfns.append(pfn)
                         unknown = False
                         break
                 if unknown:
-                    print('Cannot find any RSE associated to %s' % surl)
+                    print('Cannot find any RSE associated to %s' % pfn)
             client.add_bad_pfns(pfns=list_bad_pfns, reason=args.reason, state='BAD', expires_at=None)
             ndeclared = len(list_bad_pfns)
             tot_file_declared += ndeclared
@@ -1251,7 +1251,7 @@ def list_pfns(args):
                         logger.warning('The file has multiple parents')
                     for did in parents:
                         if did['type'] == 'DATASET':
-                            path = construct_surl(did['name'], scope, name, naming_convention=naming_convention)
+                            path = construct_non_deterministic_pfn(did['name'], scope, name, naming_convention=naming_convention)
                             pfn = ''.join([proto.attributes['scheme'],
                                            '://',
                                            proto.attributes['hostname'],

{rucio-34.6.0 → rucio-35.0.0rc2}/bin/rucio-replica-recoverer

@@ -183,8 +183,8 @@ Terminal output:
 
 2019-02-19 14:39:24,114 709 INFO  replica_recoverer[0/0]: ready to query replicas at RSEs like *MOCK*, declared as suspicious in the last 3 days at least 10 times and which are available on other RSEs.
 2019-02-19 14:39:24,124 709 INFO  replica_recoverer[0/0]: suspicious replica query took 0.0101511478424 seconds, total of 1 replicas were found. [{'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
-2019-02-19 14:39:24,125 709 INFO  replica_recoverer[0/0]: looking for replica surls.
-2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: found 1/1 surls (took 0.035572052002 seconds) - declaring them as bad replicas now.
+2019-02-19 14:39:24,125 709 INFO  replica_recoverer[0/0]: looking for replica pfns.
+2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: found 1/1 pfns (took 0.035572052002 seconds) - declaring them as bad replicas now.
 2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: ready to declare 1 bad replica(s) on MOCK_SUSPICIOUS: ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e'].
 2019-02-19 14:39:24,188 709 INFO  replica_recoverer[0/0]: finished declaring bad replicas on MOCK_SUSPICIOUS.
 2019-02-19 14:39:24,192 709 INFO  replica_recoverer[0/0]: graceful stop done

{rucio-34.6.0 → rucio-35.0.0rc2}/lib/rucio/alembicrevision.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ALEMBIC_REVISION = 'a08fa8de1545'  # the current alembic head revision
+ALEMBIC_REVISION = 'b0070f3695c8'  # the current alembic head revision

{rucio-34.6.0 → rucio-35.0.0rc2}/lib/rucio/client/baseclient.py

@@ -137,8 +137,6 @@ class BaseClient:
             self.logger.debug('No trace_host passed. Using rucio_host instead')
 
         self.list_hosts = [self.host]
-        self.account = account
-        self.vo = vo
         self.ca_cert = ca_cert
         self.auth_token = ""
         self.headers = {}
@@ -173,7 +171,9 @@ class BaseClient:
                     self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
                     self.ca_cert = True
 
-        if account is None:
+        if account is not None:
+            self.account = account
+        else:
             self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.')
             try:
                 self.account = environ['RUCIO_ACCOUNT']
@@ -183,7 +183,9 @@ class BaseClient:
                 except (NoOptionError, NoSectionError):
                     pass
 
-        if vo is None:
+        if vo is not None:
+            self.vo = vo
+        else:
             self.logger.debug('No VO passed. Trying to get it from environment variable RUCIO_VO.')
             try:
                 self.vo = environ['RUCIO_VO']
@@ -239,57 +241,62 @@ class BaseClient:
         return auth_type
 
     def _get_creds(self, creds: Optional[dict[str, Any]]) -> dict[str, Any]:
-        if self.auth_type == 'oidc':
-            if not creds:
-                creds = {}
-            # if there are default values, check if rucio.cfg does not specify them, otherwise put default
-            if 'oidc_refresh_lifetime' not in creds or creds['oidc_refresh_lifetime'] is None:
-                creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
-            if 'oidc_issuer' not in creds or creds['oidc_issuer'] is None:
-                creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
-            if 'oidc_audience' not in creds or creds['oidc_audience'] is None:
-                creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
-            if 'oidc_auto' not in creds or creds['oidc_auto'] is False:
-                creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
-            if creds['oidc_auto']:
-                if 'oidc_username' not in creds or creds['oidc_username'] is None:
-                    creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
-                if 'oidc_password' not in creds or creds['oidc_password'] is None:
-                    creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
-            if 'oidc_scope' not in creds or creds['oidc_scope'] == 'openid profile':
-                creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
-            if 'oidc_polling' not in creds or creds['oidc_polling'] is False:
-                creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)
-
-        if creds is None:
+        if not creds:
             self.logger.debug('No creds passed. Trying to get it from the config file.')
             creds = {}
-            try:
-                if self.auth_type in ['userpass', 'saml']:
+
+        try:
+            if self.auth_type == 'oidc':
+                # if there are default values, check if rucio.cfg does not specify them, otherwise put default
+                if 'oidc_refresh_lifetime' not in creds or creds['oidc_refresh_lifetime'] is None:
+                    creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
+                if 'oidc_issuer' not in creds or creds['oidc_issuer'] is None:
+                    creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
+                if 'oidc_audience' not in creds or creds['oidc_audience'] is None:
+                    creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
+                if 'oidc_auto' not in creds or creds['oidc_auto'] is False:
+                    creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
+                if creds['oidc_auto']:
+                    if 'oidc_username' not in creds or creds['oidc_username'] is None:
+                        creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
+                    if 'oidc_password' not in creds or creds['oidc_password'] is None:
+                        creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
+                if 'oidc_scope' not in creds or creds['oidc_scope'] == 'openid profile':
+                    creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
+                if 'oidc_polling' not in creds or creds['oidc_polling'] is False:
+                    creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)
+
+            elif self.auth_type in ['userpass', 'saml']:
+                if 'username' not in creds or creds['username'] is None:
                     creds['username'] = config_get('client', 'username')
+                if 'password' not in creds or creds['password'] is None:
                     creds['password'] = config_get('client', 'password')
-                elif self.auth_type == 'x509':
+
+            elif self.auth_type == 'x509':
+                if 'client_cert' not in creds or creds['client_cert'] is None:
                     if "RUCIO_CLIENT_CERT" in environ:
-                        client_cert = environ["RUCIO_CLIENT_CERT"]
+                        creds['client_cert'] = environ["RUCIO_CLIENT_CERT"]
                     else:
-                        client_cert = config_get('client', 'client_cert')
-                    creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(client_cert)))
-                    if not path.exists(creds['client_cert']):
-                        raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert'])
+                        creds['client_cert'] = config_get('client', 'client_cert')
+                creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(creds['client_cert'])))
+                if not path.exists(creds['client_cert']):
+                    raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert'])
 
+                if 'client_key' not in creds or creds['client_key'] is None:
                     if "RUCIO_CLIENT_KEY" in environ:
-                        client_key = environ["RUCIO_CLIENT_KEY"]
-                    else:
-                        client_key = config_get('client', 'client_key')
-                    creds['client_key'] = path.abspath(path.expanduser(path.expandvars(client_key)))
-                    if not path.exists(creds['client_key']):
-                        raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key'])
+                        creds['client_key'] = environ["RUCIO_CLIENT_KEY"]
                     else:
-                        perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
-                        if perms not in ['400', '600']:
-                            raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))
+                        creds['client_key'] = config_get('client', 'client_key')
+                creds['client_key'] = path.abspath(path.expanduser(path.expandvars(creds['client_key'])))
+                if not path.exists(creds['client_key']):
+                    raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key'])
+                else:
+                    perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
+                    if perms not in ['400', '600']:
+                        raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))
 
-                elif self.auth_type == 'x509_proxy':
+            elif self.auth_type == 'x509_proxy':
+                if 'client_proxy' not in creds or creds['client_proxy'] is None:
                     try:
                         creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
                     except NoOptionError:
@@ -307,11 +314,15 @@ class BaseClient:
                                 raise MissingClientParameter(
                                     'Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
                                     '\'x509_proxy\' not set in the configuration file.' % fname)
-                elif self.auth_type == 'ssh':
+
+            elif self.auth_type == 'ssh':
+                if 'ssh_private_key' not in creds or creds['ssh_private_key'] is None:
                     creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
-            except (NoOptionError, NoSectionError) as error:
-                if error.args[0] != 'client_key':
-                    raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
+
+        except (NoOptionError, NoSectionError) as error:
+            if error.args[0] != 'client_key':
+                raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
+
         return creds
 
     def _get_exception(self, headers: dict[str, str], status_code: Optional[int] = None, data=None) -> tuple[type[exception.RucioException], str]:

{rucio-34.6.0 → rucio-35.0.0rc2}/lib/rucio/client/didclient.py

@@ -21,7 +21,7 @@ from requests.status_codes import codes
 
 from rucio.client.baseclient import BaseClient, choice
 from rucio.common.exception import DeprecationError
-from rucio.common.utils import build_url, date_to_str, render_json, render_json_list
+from rucio.common.utils import build_url, date_to_str, render_json
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator, Mapping, Sequence
@@ -142,7 +142,7 @@ class DIDClient(BaseClient):
         """
         path = '/'.join([self.DIDS_BASEURL])
         url = build_url(choice(self.list_hosts), path=path)
-        r = self._send_request(url, type_='POST', data=render_json_list(dids))
+        r = self._send_request(url, type_='POST', data=render_json(dids))
         if r.status_code == codes.created:
             return True
         else: