rucio-clients 37.3.0__py3-none-any.whl → 37.4.0__py3-none-any.whl

rucio/client/baseclient.py

@@ -23,7 +23,7 @@ import secrets
 import sys
 import time
 from configparser import NoOptionError, NoSectionError
-from os import environ, fdopen, geteuid, makedirs, path
+from os import environ, fdopen, geteuid, makedirs
 from shutil import move
 from tempfile import mkstemp
 from typing import TYPE_CHECKING, Any, Optional
@@ -37,7 +37,7 @@ from requests.status_codes import codes
 
 from rucio import version
 from rucio.common import exception
-from rucio.common.config import config_get, config_get_bool, config_get_int
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_has_section
 from rucio.common.exception import CannotAuthenticate, ClientProtocolNotFound, ClientProtocolNotSupported, ConfigNotFound, MissingClientParameter, MissingModuleException, NoAuthInformation, ServerConnectionException
 from rucio.common.extra import import_extras
 from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign
@@ -73,6 +73,14 @@ def choice(hosts):
     return secrets.choice(hosts)
 
 
+def _expand_path(path: str) -> str:
+    """Fully expand path, including ~ and env variables"""
+    path = path.strip()
+    if path == '':
+        return ''
+    return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))
+
+
 class BaseClient:
 
     """Main client class for accessing Rucio resources. Handles the authentication."""
@@ -95,15 +103,29 @@ class BaseClient:
                  logger: 'Logger' = LOG) -> None:
         """
         Constructor of the BaseClient.
-        :param rucio_host: The address of the rucio server, if None it is read from the config file.
-        :param auth_host: The address of the rucio authentication server, if None it is read from the config file.
-        :param account: The account to authenticate to rucio.
-        :param ca_cert: The path to the rucio server certificate.
-        :param auth_type: The type of authentication (e.g.: 'userpass', 'kerberos' ...)
-        :param creds: Dictionary with credentials needed for authentication.
-        :param user_agent: Indicates the client.
-        :param vo: The VO to authenticate into.
-        :param logger: Logger object to use. If None, use the default LOG created by the module
+
+        Parameters
+        ----------
+        rucio_host :
+            The address of the rucio server, if None it is read from the config file.
+        auth_host :
+            The address of the rucio authentication server, if None it is read from the config file.
+        account :
+            The account to authenticate to rucio.
+        ca_cert :
+            The path to the rucio server certificate.
+        auth_type :
+            The type of authentication (e.g.: 'userpass', 'kerberos' ...)
+        creds :
+            Dictionary with credentials needed for authentication.
+        timeout :
+            Timeout for requests.
+        user_agent :
+            Indicates the client.
+        vo :
+            The VO to authenticate into.
+        logger :
+            Logger object to use. If None, use the default LOG created by the module.
         """
 
         self.logger = logger
@@ -173,7 +195,7 @@ class BaseClient:
             if self.ca_cert is None:
                 self.logger.debug('HTTPS is required, but no ca_cert was passed and X509_CERT_DIR is not defined. Trying to get it from the config file.')
                 try:
-                    self.ca_cert = path.expandvars(config_get('client', 'ca_cert'))
+                    self.ca_cert = _expand_path(config_get('client', 'ca_cert'))
                 except (NoOptionError, NoSectionError):
                     self.logger.debug('No ca_cert found in configuration. Falling back to Mozilla default CA bundle (certifi).')
                     self.ca_cert = True
@@ -289,46 +311,61 @@ class BaseClient:
                         creds['client_cert'] = environ["RUCIO_CLIENT_CERT"]
                     else:
                         creds['client_cert'] = config_get('client', 'client_cert')
-                creds['client_cert'] = path.abspath(path.expanduser(path.expandvars(creds['client_cert'])))
-                if not path.exists(creds['client_cert']):
-                    raise MissingClientParameter('X.509 client certificate not found: %s' % creds['client_cert'])
+
+                creds['client_cert'] = _expand_path(creds['client_cert'])
+
+                if not os.path.exists(creds['client_cert']):
+                    raise MissingClientParameter('X.509 client certificate not found: %r' % creds['client_cert'])
 
                 if 'client_key' not in creds or creds['client_key'] is None:
                     if "RUCIO_CLIENT_KEY" in environ:
                         creds['client_key'] = environ["RUCIO_CLIENT_KEY"]
                     else:
                         creds['client_key'] = config_get('client', 'client_key')
-                creds['client_key'] = path.abspath(path.expanduser(path.expandvars(creds['client_key'])))
-                if not path.exists(creds['client_key']):
-                    raise MissingClientParameter('X.509 client key not found: %s' % creds['client_key'])
-                else:
-                    perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
-                    if perms not in ['400', '600']:
-                        raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))
+
+                creds['client_key'] = _expand_path(creds['client_key'])
+                if not os.path.exists(creds['client_key']):
+                    raise MissingClientParameter('X.509 client key not found: %r' % creds['client_key'])
+
+                perms = oct(os.stat(creds['client_key']).st_mode)[-3:]
+                if perms not in ['400', '600']:
+                    raise CannotAuthenticate('X.509 authentication selected, but private key (%s) permissions are liberal (required: 400 or 600, found: %s)' % (creds['client_key'], perms))
 
             elif self.auth_type == 'x509_proxy':
+                # rucio specific configuration takes precedence over GSI logic
+                # environment variables take precedence over config values
+                # So we check in order:
+                # RUCIO_CLIENT_PROXY env variable
+                # client.client_x509_proxy rucio cfg variable
+                # X509_USER_PROXY env variable
+                # /tmp/x509up_u`id -u` if exists
+
+                gsi_proxy_path = '/tmp/x509up_u%d' % geteuid()
                 if 'client_proxy' not in creds or creds['client_proxy'] is None:
-                    try:
-                        creds['client_proxy'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'client_x509_proxy'))))
-                    except NoOptionError:
-                        # Recreate the classic GSI logic for locating the proxy:
-                        # - $X509_USER_PROXY, if it is set.
-                        # - /tmp/x509up_u`id -u` otherwise.
-                        # If neither exists (at this point, we don't care if it exists but is invalid), then rethrow
-                        if 'X509_USER_PROXY' in environ:
-                            creds['client_proxy'] = environ['X509_USER_PROXY']
-                        else:
-                            fname = '/tmp/x509up_u%d' % geteuid()
-                            if path.exists(fname):
-                                creds['client_proxy'] = fname
-                            else:
-                                raise MissingClientParameter(
-                                    'Cannot find a valid X509 proxy; not in %s, $X509_USER_PROXY not set, and '
-                                    '\'x509_proxy\' not set in the configuration file.' % fname)
+                    if 'RUCIO_CLIENT_PROXY' in environ:
+                        creds['client_proxy'] = environ['RUCIO_CLIENT_PROXY']
+                    elif config_has_section('client') and config_get('client', 'client_x509_proxy', default='') != '':
+                        creds['client_proxy'] = config_get('client', 'client_x509_proxy')
+                    elif 'X509_USER_PROXY' in environ:
+                        creds['client_proxy'] = environ['X509_USER_PROXY']
+                    elif os.path.isfile(gsi_proxy_path):
+                        creds['client_proxy'] = gsi_proxy_path
+
+                creds['client_proxy'] = _expand_path(creds['client_proxy'])
+
+                if not os.path.isfile(creds['client_proxy']):
+                    raise MissingClientParameter(
+                        'Cannot find a valid X509 proxy; checked $RUCIO_CLIENT_PROXY, $X509_USER_PROXY'
+                        'client/client_x509_proxy config and default path: %r' % gsi_proxy_path
+                    )
 
             elif self.auth_type == 'ssh':
                 if 'ssh_private_key' not in creds or creds['ssh_private_key'] is None:
-                    creds['ssh_private_key'] = path.abspath(path.expanduser(path.expandvars(config_get('client', 'ssh_private_key'))))
+                    creds['ssh_private_key'] = config_get('client', 'ssh_private_key')
+
+                creds['ssh_private_key'] = _expand_path(creds['ssh_private_key'])
+                if not os.path.isfile(creds["ssh_private_key"]):
+                    raise CannotAuthenticate('Provided ssh private key %r does not exist' % creds['ssh_private_key'])
 
         except (NoOptionError, NoSectionError) as error:
             if error.args[0] != 'client_key':
@@ -535,11 +572,11 @@ class BaseClient:
 
         if not self.auth_oidc_refresh_active:
             return False
-        if path.exists(self.token_exp_epoch_file):
+        if os.path.exists(self.token_exp_epoch_file):
             with open(self.token_exp_epoch_file, 'r') as token_epoch_file:
                 try:
                     self.token_exp_epoch = int(token_epoch_file.readline())
-                except:
+                except Exception:
                     self.token_exp_epoch = None
 
         if self.token_exp_epoch is None:
@@ -718,10 +755,10 @@ class BaseClient:
             url = build_url(self.auth_host, path='auth/x509_proxy')
             client_cert = self.creds['client_proxy']
 
-        if (client_cert is not None) and not (path.exists(client_cert)):
+        if (client_cert is not None) and not (os.path.exists(client_cert)):
             self.logger.error('given client cert (%s) doesn\'t exist' % client_cert)
             return False
-        if client_key is not None and not path.exists(client_key):
+        if client_key is not None and not os.path.exists(client_key):
             self.logger.error('given client key (%s) doesn\'t exist' % client_key)
 
         if client_key is None:
@@ -755,10 +792,10 @@ class BaseClient:
         headers = {}
 
         private_key_path = self.creds['ssh_private_key']
-        if not path.exists(private_key_path):
+        if not os.path.exists(private_key_path):
             self.logger.error('given private key (%s) doesn\'t exist' % private_key_path)
             return False
-        if private_key_path is not None and not path.exists(private_key_path):
+        if private_key_path is not None and not os.path.exists(private_key_path):
             self.logger.error('given private key (%s) doesn\'t exist' % private_key_path)
             return False
 
@@ -906,7 +943,7 @@ class BaseClient:
 
         :return: True if a token could be read. False if no file exists.
         """
-        if not path.exists(self.token_file):
+        if not os.path.exists(self.token_file):
             return False
 
         try:
@@ -927,7 +964,7 @@ class BaseClient:
         Write the current auth_token to the local token file.
         """
         # check if rucio temp directory is there. If not create it with permissions only for the current user
-        if not path.isdir(self.token_path):
+        if not os.path.isdir(self.token_path):
             try:
                 self.logger.debug('rucio token folder \'%s\' not found. Create it.' % self.token_path)
                 makedirs(self.token_path, 0o700)

rucio/client/client.py

@@ -58,63 +58,71 @@ class Client(AccountClient,
              LifetimeClient):
 
     """
-        Main client class for accessing Rucio resources. Handles the authentication.
+    Main client class for accessing Rucio resources. Handles the authentication.
 
-        Note:
-            Used to access all client methods. Each entity client *can* be used to access methods, but using the main client class is recommended for ease of use.
+    Note:
+    ------
+        Used to access all client methods. Each entity client *can* be used to access methods, but using the main client class is recommended for ease of use.
 
-    For using general methods -
 
+    Example:
+    -------
+        from rucio.client import Client
 
-    ```
-    from rucio.client import Client
+        client = Client()  # authenticate with config or environ settings
+        client.add_replication_rule(...)
 
-    client = Client()  # authenticate with config or environ settings
-    client.add_replication_rule(...)
+        client = Client(
+            rucio_host = "my_host",
+            auth_host = "my_auth_host",
+            account = "jdoe12345",
+            auth_type = "userpass",
+            creds = {
+                "username": "jdoe12345",
+                "password": "******",
+            }
+        ) # authenticate with kwargs
+        client.list_replicas(...)
 
-    client = Client(
-        rucio_host = "my_host",
-        auth_host = "my_auth_host",
-        account = "jdoe12345",
-        auth_type = "userpass",
-        creds = {
-            "username": "jdoe12345",
-            "password": "******",
-        }
-    ) # authenticate with kwargs
-    client.list_replicas(...)
-    ```
 
-    For using the upload and download clients -
+        # For using the upload and download clients
 
-    ```
-    from rucio.client import Client
-    from rucio.client.uploadclient import UploadClient
-    from rucio.client.downloadclient import DownloadClient
+        from rucio.client import Client
+        from rucio.client.uploadclient import UploadClient
+        from rucio.client.downloadclient import DownloadClient
 
-    client = Client(...) # Initialize a client using your preferred method
+        client = Client(...) # Initialize a client using your preferred method
 
-    upload_client = UploadClient(client) # Pass forward the initialized client
-    upload_client.upload(items=...)
-
-    download_client = DownloadClient(client)
-    download_client.download_dids(items=...)
-    ```
+        upload_client = UploadClient(client) # Pass forward the initialized client
+        upload_client.upload(items=...)
 
+        download_client = DownloadClient(client)
+        download_client.download_dids(items=...)
     """
 
     def __init__(self, **args):
         """
         Constructor for the Rucio main client class.
 
-        :param rucio_host: the host of the rucio system.
-        :param auth_host: the host of the rucio authentication server.
-        :param account: the rucio account that should be used to interact with the rucio system.
-        :param ca_cert: the certificate to verify the server.
-        :param auth_type: the type of authentication to use (e.g. userpass, x509 ...)
-        :param creds: credentials needed for authentication.
-        :param timeout: Float describes the timeout of the request (in seconds).
-        :param vo: The vo that the client will interact with.
-        :param logger: Logger instance to use (optional)
+        Parameters
+        ----------
+        rucio_host :
+            The host of the rucio system.
+        auth_host :
+            The host of the rucio authentication server.
+        account :
+            The rucio account that should be used to interact with the rucio system.
+        ca_cert :
+            The certificate to verify the server.
+        auth_type :
+            The type of authentication to use (e.g. userpass, x509 ...).
+        creds :
+            Credentials needed for authentication.
+        timeout :
+            Describes the timeout of the request (in seconds).
+        vo :
+            The vo that the client will interact with.
+        logger :
+            Logger instance to use.
         """
         super(Client, self).__init__(**args)

rucio/client/configclient.py

@@ -35,9 +35,12 @@ class ConfigClient(BaseClient):
         """
         Sends the request to get the matching configuration.
 
-        :param section: the optional name of the section.
-        :param option: the optional option within the section.
-        :return: dictionary containing the configuration.
+        Parameters
+        ----------
+        section :
+            The name of the section.
+        option :
+            The option within the section.
         """
 
         if section is None and option is not None:
@@ -68,18 +71,30 @@ class ConfigClient(BaseClient):
         """
         Sends the request to create or set an option within a section. Missing sections will be created.
 
-        :param section: the name of the section.
-        :param option: the name of the option.
-        :param value: the value to set on the config option
-        :param use_body_for_params: send parameters in a json-encoded request body instead of url-encoded
-        TODO: remove this parameter
+        Parameters
+        ----------
+        section :
+            The name of the section.
+        option :
+            The name of the option.
+        value :
+            The value to set on the config option.
+        use_body_for_params :
+            Send parameters in a json-encoded request body instead of url-encoded.
+
+        Returns
+        -------
+        bool
+            True if option was set successfully.
+
+        Note:
+        ------
         The format of the /config endpoint was recently changed. We migrated from performing a PUT on
         "/config/<section>/<option>/<value>" to sending the parameters using a json-encoded body.
         This was done to fix multiple un-wanted side effects related to how the middleware treats
         values encoded in a path.
         For a smooth transition, we allow both cases for now, but we should migrate to only passing
         values via the request body.
-        :return: True if option was removed successfully. False otherwise.
         """
 
         if use_body_for_params:
@@ -107,11 +122,19 @@ class ConfigClient(BaseClient):
             option: str
     ) -> bool:
         """
-        Sends the request to remove an option from a section
+        Sends the request to remove an option from a section.
 
-        :param section: the name of the section.
-        :param option: the name of the option.
-        :return: True if option was removed successfully. False otherwise.
+        Parameters
+        ----------
+        section :
+            The name of the section.
+        option :
+            The name of the option.
+
+        Returns
+        -------
+
+            True if option was removed successfully.
         """
 
         path = '/'.join([self.CONFIG_BASEURL, section, option])

rucio/client/credentialclient.py

@@ -34,13 +34,23 @@ class CredentialClient(BaseClient):
         """
         Return a signed version of the given URL for the given operation.
 
-        :param rse: The name of the RSE the URL points to.
-        :param service: The service the URL points to (gcs, s3, swift)
-        :param operation: The desired operation (read, write, delete)
-        :param url: The URL to sign
-        :param lifetime: The desired lifetime of the URL in seconds
+        Parameters
+        ----------
+        rse :
+            The name of the RSE the URL points to.
+        service :
+            The service the URL points to (gcs, s3, swift)
+        operation :
+            The desired operation (read, write, delete)
+        url :
+            The URL to sign
+        lifetime :
+            The desired lifetime of the URL in seconds, by default 3600
 
-        :return: The signed URL string
+        Returns
+        -------
+
+            The signed URL string
         """
         path = '/'.join([self.CREDENTIAL_BASEURL, 'signurl'])
         params = {}