rucio-clients 34.5.0__tar.gz → 35.0.0rc1__tar.gz

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/AUTHORS.rst

@@ -66,6 +66,7 @@ Individual contributors to the source code
 - Jensen Zhang <hack@jensen-zhang.site>, 2022
 - Aksel Lunde Aase <aksel.lunde.aase@gmail.com>, 2022
 - Elena Gazzarrini <gazzarrini.elena@gmail.com>, 2022-2023
+- Maximilian Linhoff, <maximilian.linhoff@tu-dortmund.de>, 2024
 
 Organisations employing contributors
 ------------------------------------
@@ -93,3 +94,4 @@ Organisations employing contributors
 - Imperial College London (UK)
 - University of Chicago (USA)
 - Purdue University (USA)
+- TU Dortmund University (Germany)

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/MANIFEST.in

@@ -7,7 +7,7 @@ include setuputil.py
 include setup.py
 include setup.cfg
 include pyproject.toml
-include requirements.txt
+include requirements/requirements.client.txt
 include etc/rse-accounts.cfg.template
 include etc/rucio.cfg.atlas.client.template
 include etc/rucio.cfg.template

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 34.5.0
+Version: 35.0.0rc1
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio
@@ -19,22 +19,22 @@ Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.9, <4
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: requests~=2.32.0
-Requires-Dist: urllib3~=1.26.18
-Requires-Dist: dogpile.cache~=1.2.2
-Requires-Dist: tabulate~=0.9.0
-Requires-Dist: jsonschema~=4.20.0
+Requires-Dist: requests>=2.32.2
+Requires-Dist: urllib3>=1.26.18
+Requires-Dist: dogpile.cache>=1.2.2
+Requires-Dist: tabulate>=0.9.0
+Requires-Dist: jsonschema>=4.20.0
 Provides-Extra: ssh
-Requires-Dist: paramiko~=3.4.0; extra == "ssh"
+Requires-Dist: paramiko>=3.4.0; extra == "ssh"
 Provides-Extra: kerberos
-Requires-Dist: kerberos~=1.3.1; extra == "kerberos"
-Requires-Dist: pykerberos~=1.2.4; extra == "kerberos"
+Requires-Dist: kerberos>=1.3.1; extra == "kerberos"
+Requires-Dist: pykerberos>=1.2.4; extra == "kerberos"
 Requires-Dist: requests-kerberos>=0.14.0; extra == "kerberos"
 Provides-Extra: swift
-Requires-Dist: python-swiftclient~=4.4.0; extra == "swift"
+Requires-Dist: python-swiftclient>=4.4.0; extra == "swift"
 Provides-Extra: argcomplete
-Requires-Dist: argcomplete~=3.1.6; extra == "argcomplete"
+Requires-Dist: argcomplete>=3.1.6; extra == "argcomplete"
 Provides-Extra: sftp
-Requires-Dist: paramiko~=3.4.0; extra == "sftp"
+Requires-Dist: paramiko>=3.4.0; extra == "sftp"
 Provides-Extra: dumper
-Requires-Dist: python-magic~=0.4.27; extra == "dumper"
+Requires-Dist: python-magic>=0.4.27; extra == "dumper"

rucio_clients-35.0.0rc1/README.md

@@ -0,0 +1,27 @@
+# Rucio - Scientific Data Management
+
+Rucio is a software framework that provides functionality to organize, manage, and access large volumes of scientific data using customisable policies.
+The data can be spread across globally distributed locations and across heterogeneous data centers, uniting different storage and network technologies as a single federated entity.
+Rucio offers advanced features such as distributed data recovery or adaptive replication, and is highly scalable, modular, and extensible.
+Rucio has been originally developed to meet the requirements of the high-energy physics experiment ATLAS, and is continuously extended to support LHC experiments and other diverse scientific communities.
+
+## Documentation
+
+General information, API/REST description and guides can be found in our [documentation](https://rucio.cern.ch/documentation) or on our [webpage](https://rucio.cern.ch).
+
+## Try it out
+
+We provide a [dockerized environment](https://github.com/rucio/rucio/tree/master/etc/docker/dev) which serves both as a demo environment and a development environment.
+It includes all the necessary preconfigured components for multiple storage and transfers developments.
+
+## Developers
+
+For information on how to contribute to Rucio, please refer and follow our [CONTRIBUTING](https://rucio.cern.ch/documentation/contributing) guidelines. We strongly recommend to use the [dockerized environment](https://github.com/rucio/rucio/tree/master/etc/docker/dev) for development.
+
+## Operators
+
+To learn how to deploy and configure Rucio, consult the [documentation](https://rucio.cern.ch/documentation) available online.
+
+## Getting Support
+
+If you are looking for support, please contact us via one of our [official channels](https://rucio.cern.ch/documentation/contact_us/).

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/bin/rucio

@@ -30,6 +30,7 @@ from configparser import NoOptionError, NoSectionError
 from copy import deepcopy
 from datetime import datetime
 from functools import wraps
+from typing import Optional
 
 from tabulate import tabulate
 
@@ -1079,6 +1080,24 @@ def download(args):
             logger.debug(args.dids)
         item_defaults['pfn'] = args.pfn
         item_defaults['did'] = did_str
+        if args.rses is None:
+            logger.warning("No RSE was given, selecting one.")
+            client = get_client(args)
+            replicas = client.list_replicas(
+                [{"scope": did_str.split(':')[0], "name": did_str.split(':')[-1]}],
+                schemes=args.protocol,
+                ignore_availability=False,
+                client_location=detect_client_location(),
+                resolve_archives=not args.no_resolve_archives
+            )
+
+            download_rse = _get_rse_for_pfn(replicas, args.pfn)
+            if download_rse is None:
+                logger.error("Could not find RSE for pfn %s", args.pfn)
+                return FAILURE
+            else:
+                item_defaults['rse'] = download_rse
+
         result = download_client.download_pfns([item_defaults], 1, trace_pattern, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
 
     if not result:
@@ -1120,6 +1139,21 @@ def download(args):
     return SUCCESS
 
 
+def _get_rse_for_pfn(replicas, pfn) -> Optional[str]:
+    # Check each rse in the replica list for the pfn. If no pfn is found, returns None.
+    # If it is found, stop the generator and return the item.
+    for replica in replicas:
+        try:
+            download_rse = next(
+                rse for rse in replica['rses']
+                if pfn in replica['rses'][rse]
+            )
+        except StopIteration:
+            continue
+        else:
+            return download_rse
+
+
 @exception_handler
 def get_metadata(args):
     """
@@ -1929,7 +1963,7 @@ To list the missing replica of a dataset of a given RSE-expression::
     list_file_replicas_parser.add_argument('--no-resolve-archives', dest='no_resolve_archives', default=False, action='store_true', help='Do not resolve archives which may contain the files.', required=False)
     list_file_replicas_parser.add_argument('--sort', dest='sort', default=None, action='store', help='Replica sort algorithm. Available options: geoip (default), random', required=False)
     list_file_replicas_parser.add_argument('--rses', dest='rses', default=None, action='store', help='The RSE filter expression. A comprehensive help about RSE expressions\
-            can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_expressions/' + Color.END)
+            can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
 
     # The list-dataset-replicas command
     list_dataset_replicas_parser = subparsers.add_parser('list-dataset-replicas', help='List the dataset replicas.',
@@ -2431,13 +2465,13 @@ You can filter by account::
     list_rses_parser = subparsers.add_parser('list-rses', help='Show the list of all the registered Rucio Storage Elements (RSEs).')
     list_rses_parser.set_defaults(function=list_rses)
     list_rses_parser.add_argument('--rses', dest='rses', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
-can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_expressions/' + Color.END)
+can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
 
     # The list-suspicious-replicas command
     list_suspicious_replicas_parser = subparsers.add_parser('list-suspicious-replicas', help='Show the list of all replicas marked "suspicious".')
     list_suspicious_replicas_parser.set_defaults(function=list_suspicious_replicas)
     list_suspicious_replicas_parser.add_argument('--expression', dest='rse_expression', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
-can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_expressions/' + Color.END)
+can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
     list_suspicious_replicas_parser.add_argument('--younger_than', '--younger-than', new_option_string='--younger-than', dest='younger_than', action=StoreAndDeprecateWarningAction, help='List files that have been marked suspicious since the date "younger_than", e.g. 2021-11-29T00:00:00.')  # NOQA: E501
     list_suspicious_replicas_parser.add_argument('--nattempts', dest='nattempts', action='store', help='Minimum number of failed attempts to access a suspicious file.')
 

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/bin/rucio-admin

@@ -54,7 +54,7 @@ from rucio.common.exception import (
     ScopeNotFound,
 )
 from rucio.common.extra import import_extras
-from rucio.common.utils import StoreAndDeprecateWarningAction, chunks, clean_surls, construct_surl, extract_scope, get_bytes_value_from_string, parse_response, render_json, sizefmt
+from rucio.common.utils import StoreAndDeprecateWarningAction, chunks, clean_pfns, construct_non_deterministic_pfn, extract_scope, get_bytes_value_from_string, parse_response, render_json, sizefmt
 from rucio.rse import rsemanager as rsemgr
 
 EXTRA_MODULES = import_extras(['argcomplete'])
@@ -1148,21 +1148,21 @@ def declare_bad_file_replicas(args):
             list_bad_pfns = []
             cnt += 1
             previous_pattern = None
-            for surl in clean_surls(chunk):
+            for pfn in clean_pfns(chunk):
                 unknown = True
                 if previous_pattern:
-                    if previous_pattern in surl:
-                        list_bad_pfns.append(surl)
+                    if previous_pattern in pfn:
+                        list_bad_pfns.append(pfn)
                         unknown = False
                         continue
                 for pattern in prot_dict:
-                    if pattern in surl:
+                    if pattern in pfn:
                         previous_pattern = prot_dict[pattern]
-                        list_bad_pfns.append(surl)
+                        list_bad_pfns.append(pfn)
                         unknown = False
                         break
                 if unknown:
-                    print('Cannot find any RSE associated to %s' % surl)
+                    print('Cannot find any RSE associated to %s' % pfn)
             client.add_bad_pfns(pfns=list_bad_pfns, reason=args.reason, state='BAD', expires_at=None)
             ndeclared = len(list_bad_pfns)
             tot_file_declared += ndeclared
@@ -1251,7 +1251,7 @@ def list_pfns(args):
                         logger.warning('The file has multiple parents')
                     for did in parents:
                         if did['type'] == 'DATASET':
-                            path = construct_surl(did['name'], scope, name, naming_convention=naming_convention)
+                            path = construct_non_deterministic_pfn(did['name'], scope, name, naming_convention=naming_convention)
                             pfn = ''.join([proto.attributes['scheme'],
                                            '://',
                                            proto.attributes['hostname'],

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/lib/rucio/alembicrevision.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ALEMBIC_REVISION = 'a08fa8de1545'  # the current alembic head revision
+ALEMBIC_REVISION = 'b0070f3695c8'  # the current alembic head revision

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/lib/rucio/client/baseclient.py

@@ -137,8 +137,6 @@ class BaseClient:
             self.logger.debug('No trace_host passed. Using rucio_host instead')
 
         self.list_hosts = [self.host]
-        self.account = account
-        self.vo = vo
         self.ca_cert = ca_cert
         self.auth_token = ""
         self.headers = {}
@@ -173,7 +171,9 @@ class BaseClient:
                     self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
                     self.ca_cert = True
 
-        if account is None:
+        if account is not None:
+            self.account = account
+        else:
             self.logger.debug('No account passed. Trying to get it from the RUCIO_ACCOUNT environment variable or the config file.')
             try:
                 self.account = environ['RUCIO_ACCOUNT']
@@ -183,7 +183,9 @@ class BaseClient:
                 except (NoOptionError, NoSectionError):
                     pass
 
-        if vo is None:
+        if vo is not None:
+            self.vo = vo
+        else:
             self.logger.debug('No VO passed. Trying to get it from environment variable RUCIO_VO.')
             try:
                 self.vo = environ['RUCIO_VO']
@@ -239,57 +241,62 @@ class BaseClient:
         return auth_type
 
     def _get_creds(self, creds: Optional[dict[str, Any]]) -> dict[str, Any]:
-        if self.auth_type == 'oidc':
-            if not creds:
-                creds = {}
-            # if there are default values, check if rucio.cfg does not specify them, otherwise put default
-            if 'oidc_refresh_lifetime' not in creds or creds['oidc_refresh_lifetime'] is None:
-                creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
-            if 'oidc_issuer' not in creds or creds['oidc_issuer'] is None:
-                creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
-            if 'oidc_audience' not in creds or creds['oidc_audience'] is None:
-                creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
-            if 'oidc_auto' not in creds or creds['oidc_auto'] is False:
-                creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
-            if creds['oidc_auto']:
-                if 'oidc_username' not in creds or creds['oidc_username'] is None:
-                    creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
-                if 'oidc_password' not in creds or creds['oidc_password'] is None:
-                    creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
-            if 'oidc_scope' not in creds or creds['oidc_scope'] == 'openid profile':
-                creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
-            if 'oidc_polling' not in creds or creds['oidc_polling'] is False:
-                creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)
-
-        if creds is None:
+        if not creds:
             self.logger.debug('No creds passed. Trying to get it from the config file.')
             creds = {}
-            try:
-                if self.auth_type in ['userpass', 'saml']:
+
+        try:
+            if self.auth_type == 'oidc':
+                # if there are default values, check if rucio.cfg does not specify them, otherwise put default
+                if 'oidc_refresh_lifetime' not in creds or creds['oidc_refresh_lifetime'] is None:
+                    creds['oidc_refresh_lifetime'] = config_get('client', 'oidc_refresh_lifetime', False, None)
+                if 'oidc_issuer' not in creds or creds['oidc_issuer'] is None:
+                    creds['oidc_issuer'] = config_get('client', 'oidc_issuer', False, None)
+                if 'oidc_audience' not in creds or creds['oidc_audience'] is None:
+                    creds['oidc_audience'] = config_get('client', 'oidc_audience', False, None)
+                if 'oidc_auto' not in creds or creds['oidc_auto'] is False:
+                    creds['oidc_auto'] = config_get_bool('client', 'oidc_auto', False, False)
+                if creds['oidc_auto']:
+                    if 'oidc_username' not in creds or creds['oidc_username'] is None:
+                        creds['oidc_username'] = config_get('client', 'oidc_username', False, None)
+                    if 'oidc_password' not in creds or creds['oidc_password'] is None:
+                        creds['oidc_password'] = config_get('client', 'oidc_password', False, None)
+                if 'oidc_scope' not in creds or creds['oidc_scope'] == 'openid profile':
+                    creds['oidc_scope'] = config_get('client', 'oidc_scope', False, 'openid profile')
+                if 'oidc_polling' not in creds or creds['oidc_polling'] is False:
+                    creds['oidc_polling'] = config_get_bool('client', 'oidc_polling', False, False)
+
+            elif self.auth_type in ['userpass', 'saml']:
+                if 'username' not in creds or creds['username'] is None:
                     creds['username'] = config_get('client', 'username')
+                if 'password' not in creds or creds['password'] is None:
                     creds['password'] = config_get('client', 'password')
-                elif self.auth_type == 'x509':
+
+            elif self.auth_type == 'x509':
+                if 'client_cert' not in creds or creds['client_cert'] is None:
                     if "RUCIO_CLIENT_CERT" in environ:
-                        client_cert = environ["RUCIO_CLIENT_CERT"]
+                        creds['client_cert'] = environ["RUCIO_CLIENT_CERT"]
                     else:
-                        client_cert = config_get('client', 'client_cert')
-                    creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(client_cert)))
-                    if not path.exists(creds['client_cert']):
-                        raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert'])
+                        creds['client_cert'] = config_get('client', 'client_cert')
+                creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(creds['client_cert'])))
+                if not path.exists(creds['client_cert']):
+                    raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert'])
 
+                if 'client_key' not in creds or creds['client_key'] is None:
                     if "RUCIO_CLIENT_KEY" in environ:
-                        client_key = environ["RUCIO_CLIENT_KEY"]
-                    else:
-                        client_key = config_get('client', 'client_key')
-                    creds['client_key'] = path.abspath(path.expanduser(path.expandvars(client_key)))
-                    if not path.exists(creds['client_key']):
-                        raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key'])
+                        creds['client_key'] = environ["RUCIO_CLIENT_KEY"]
                     else:
-                        perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
-                        if perms not in ['400', '600']:
-                            raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))
+                        creds['client_key'] = config_get('client', 'client_key')
+                creds['client_key'] = path.abspath(path.expanduser(path.expandvars(creds['client_key'])))
+                if not path.exists(creds['client_key']):
+                    raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key'])
+                else:
+                    perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
+                    if perms not in ['400', '600']:
+                        raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))
 
-                elif self.auth_type == 'x509_proxy':
+            elif self.auth_type == 'x509_proxy':
+                if 'client_proxy' not in creds or creds['client_proxy'] is None:
                     try:
                         creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
                     except NoOptionError:
@@ -307,11 +314,15 @@ class BaseClient:
                                 raise MissingClientParameter(
                                     'Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
                                     '\'x509_proxy\' not set in the configuration file.' % fname)
-                elif self.auth_type == 'ssh':
+
+            elif self.auth_type == 'ssh':
+                if 'ssh_private_key' not in creds or creds['ssh_private_key'] is None:
                     creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
-            except (NoOptionError, NoSectionError) as error:
-                if error.args[0] != 'client_key':
-                    raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
+
+        except (NoOptionError, NoSectionError) as error:
+            if error.args[0] != 'client_key':
+                raise MissingClientParameter('Option \'%s\' cannot be found in config file' % error.args[0])
+
         return creds
 
     def _get_exception(self, headers: dict[str, str], status_code: Optional[int] = None, data=None) -> tuple[type[exception.RucioException], str]:

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/lib/rucio/client/didclient.py

@@ -21,7 +21,7 @@ from requests.status_codes import codes
 
 from rucio.client.baseclient import BaseClient, choice
 from rucio.common.exception import DeprecationError
-from rucio.common.utils import build_url, date_to_str, render_json, render_json_list
+from rucio.common.utils import build_url, date_to_str, render_json
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator, Mapping, Sequence
@@ -142,7 +142,7 @@ class DIDClient(BaseClient):
         """
         path = '/'.join([self.DIDS_BASEURL])
         url = build_url(choice(self.list_hosts), path=path)
-        r = self._send_request(url, type_='POST', data=render_json_list(dids))
+        r = self._send_request(url, type_='POST', data=render_json(dids))
         if r.status_code == codes.created:
             return True
         else:

{rucio_clients-34.5.0 → rucio_clients-35.0.0rc1}/lib/rucio/client/diracclient.py

@@ -13,12 +13,16 @@
 # limitations under the License.
 
 from json import dumps
+from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from requests.status_codes import codes
 
 from rucio.client.baseclient import BaseClient, choice
 from rucio.common.utils import build_url
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
+
 
 class DiracClient(BaseClient):
 
@@ -26,7 +30,12 @@ class DiracClient(BaseClient):
 
     DIRAC_BASEURL = 'dirac'
 
-    def add_files(self, lfns, ignore_availability=False, parents_metadata=None):
+    def add_files(
+            self,
+            lfns: "Iterable[Mapping[str, Any]]",
+            ignore_availability: bool = False,
+            parents_metadata: Optional["Mapping[str, Mapping[str, Any]]"] = None
+    ) -> Literal[True]:
         """
         Bulk add files :
         - Create the file and replica.