rucio-clients 36.3.0__tar.gz → 36.5.0__tar.gz

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/AUTHORS.rst

@@ -68,6 +68,7 @@ Individual contributors to the source code
 - Elena Gazzarrini <gazzarrini.elena@gmail.com>, 2022-2023
 - Maximilian Linhoff, <maximilian.linhoff@tu-dortmund.de>, 2024
 - Eric Banzuzi, <eric.banzuzi@gmail.com>, 2024
+- Paul Millar, <paul.millar@desy.de>, 2025
 
 Organisations employing contributors
 ------------------------------------
@@ -96,3 +97,4 @@ Organisations employing contributors
 - University of Chicago (USA)
 - Purdue University (USA)
 - TU Dortmund University (Germany)
+- Deutsches Elektronen-Synchrotron DESY (Germany)

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rucio-clients
-Version: 36.3.0
+Version: 36.5.0
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/bin/rucio

@@ -90,29 +90,33 @@ def map_legacy_command() -> Optional[list[str]]:
             "touch": ["did", "update", "--touch"],
             "add-lifetime-exception": ["lifetime-exception", "add"],
         }
-        try:
-            new_command = command_map[command]
-        except KeyError:
-            pass
+        new_command = command_map.get(command)
 
     return new_command
 
 
 if __name__ == "__main__":
-    commands = ("-h", "--help", "--version", "account", "config", "did", "replica", "rse", "rule", "scope", "subscription", "ping", "whoami", "test-server", "lifetime-exception", "upload", "download")
-    logger = setup_logger(module_name=__name__)
-    first_command = _get_first_command(sys.argv[1:])
+    commands = ("account", "config", "did", "replica", "rse", "rule", "scope", "subscription", "ping", "whoami", "test-server", "lifetime-exception", "upload", "download")
 
-    is_legacy = '--legacy' in sys.argv
+    parser = argparse.ArgumentParser(add_help=False)
+    # Check for legacy flag
+    parser.add_argument("--legacy", action="store_true")
+    # Check for commands in the new command list
+    parser.add_argument("-h", "--help", action="store_true")
+    parser.add_argument("--version", action="store_true")
 
-    if (first_command in commands) and not is_legacy:
-        main()
+    args, _ = parser.parse_known_args()
 
-    elif is_legacy:
+    logger = setup_logger(module_name=__name__)
+
+    if args.legacy:
         make_warning(logger)
         sys.argv.pop(sys.argv.index('--legacy'))
         main_legacy()
 
+    elif (any(arg in commands for arg in sys.argv)) or args.help or args.version:
+        main()
+
     else:
         make_warning(logger)
         try:

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/client/commands/account.py

@@ -59,6 +59,7 @@ class Account(CommandBase):
         parser.add_argument("-a", "--account", dest="account", help="Account name")
         parser.add_argument("--id", dest="identity", action="store", help="Identity (e.g. DN)")
         parser.add_argument("--filters", dest="filters", action="store", help="Filter arguments in form `key=value,another_key=next_value`")
+        parser.add_argument("--csv", action='store_true', help="Output list of accounts as a csv")
 
     def add_namespace(self, parser: "ArgumentParser") -> None:
         parser.add_argument("--type", dest="account_type", help="Account Type", choices={"USER", "GROUP", "SERVICE"}, required=True)

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/client/commands/bin_legacy/rucio.py

@@ -1671,7 +1671,9 @@ def list_rses(args, client, logger, console, spinner):
         spinner.start()
 
     rses = client.list_rses(args.rses)
-    if cli_config == 'rich':
+    if args.csv:
+        print(*(rse['rse'] for rse in rses), sep='\n')
+    elif cli_config == 'rich':
         table = generate_table([[rse['rse']] for rse in sorted(rses, key=lambda elem: elem['rse'])], headers=['RSE'], col_alignments=['left'])
         spinner.stop()
         print_output(table, console=console, no_pager=args.no_pager)
@@ -1985,7 +1987,8 @@ def add_lifetime_exception(args, client, logger, console, spinner):
         logger.error('inputfile is mandatory')
         return FAILURE
     with open(args.inputfile) as infile:
-        dids = list(set(line.strip() for line in infile))
+        # Deduplicate the content of the input file and ignore empty lines.
+        dids = set(did for line in infile if (did := line.strip()))
 
     dids_list = []
     containers = []
@@ -2040,6 +2043,7 @@ def add_lifetime_exception(args, client, logger, console, spinner):
                     else:
                         logger.info('%s:%s will be declared' % (scope, name))
                         datasets.append({'scope': scope, 'name': name})
+                        summary[4] += 1
 
     if not datasets:
         logger.error('Nothing to submit')
@@ -2713,6 +2717,7 @@ You can filter by account::
     list_rses_parser.set_defaults(function=list_rses)
     list_rses_parser.add_argument('--rses', dest='rses', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
 can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
+    list_rses_parser.add_argument("--csv", action='store_true', help='Output a list of RSEs as a csv')
 
     # The list-suspicious-replicas command
     list_suspicious_replicas_parser = subparsers.add_parser('list-suspicious-replicas', help='Show the list of all replicas marked "suspicious".')

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/client/commands/bin_legacy/rucio_admin.py

@@ -151,7 +151,9 @@ def list_accounts(args, client, logger, console, spinner):
         for key, value in [(_.split('=')[0], _.split('=')[1]) for _ in args.filters.split(',')]:
             filters[key] = value
     accounts = client.list_accounts(identity=args.identity, account_type=args.account_type, filters=filters)
-    if cli_config == 'rich':
+    if args.csv:
+        print(*(account['account'] for account in accounts), sep=',')
+    elif cli_config == 'rich':
         table = generate_table([[account['account']] for account in accounts], headers=['ACCOUNT'], col_alignments=['left'])
         spinner.stop()
         print_output(table, console=console, no_pager=args.no_pager)
@@ -327,7 +329,9 @@ def list_rses(args, client, logger, console, spinner):
         spinner.start()
 
     rses = client.list_rses()
-    if cli_config == 'rich':
+    if args.csv:
+        print(*(rse['rse'] for rse in rses), sep='\n')
+    elif cli_config == 'rich':
         table_data = [[rse['rse']] for rse in sorted(rses, key=lambda elem: elem['rse'])]
         table = generate_table(table_data, headers=['RSE'], col_alignments=['left'])
         spinner.stop()
@@ -1455,6 +1459,7 @@ def get_parser():
     list_account_parser.add_argument('--type', dest='account_type', action='store', help='Account Type (USER, GROUP, SERVICE)')
     list_account_parser.add_argument('--id', dest='identity', action='store', help='Identity (e.g. DN)')
     list_account_parser.add_argument('--filters', dest='filters', action='store', help='Filter arguments in form `key=value,another_key=next_value`')
+    list_account_parser.add_argument("--csv", action='store_true', help='List result as a csv')
     list_account_parser.set_defaults(which='list_accounts')
 
     # The list_account_attributes command
@@ -1753,6 +1758,7 @@ def get_parser():
                                                       '\n'
                                                       '    $ rucio list-rses --rses \"tier=2&type=DATADISK\"\n'
                                                       '\n')
+    list_rse_parser.add_argument("--csv", action='store_true', help='Output a list of RSEs as a csv')
     list_rse_parser.set_defaults(which='list_rses')
 
     # The add_rse command

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/client/commands/rse.py

@@ -83,9 +83,11 @@ class RSE(CommandBase):
 
     def list_namespace(self, parser: "ArgumentParser") -> None:
         parser.add_argument("--rses", "--rse-exp", dest="rses", help="RSE name or expression")
+        parser.add_argument("--csv", action='store_true', help="Output list of RSEs as a csv")
 
     def show_namespace(self, parser: "ArgumentParser") -> None:
         parser.add_argument("--rse", "--rse-name", help="RSE name", required=True)
+        parser.add_argument("--csv", action='store_true', help="Output list of RSE property key and values as a csv")
 
     def list_(self):
         list_rses(self.args, self.client, self.logger, self.console, self.spinner)
@@ -218,7 +220,7 @@ class Protocol(RSE):
 
     def usage_example(self) -> list[str]:
         return [
-            "$ rucio rse protocol --hostname jdoes.test.org --scheme gsiftp --prefix '/atlasdatadisk/rucio/' --port 8443 --rse JDOE_DATADISK  # Add a new protocol on jdoe.test.org that uses gsiftp",
+            "$ rucio rse protocol --host jdoes.test.org --scheme gsiftp --prefix '/atlasdatadisk/rucio/' --port 8443 --rse JDOE_DATADISK  # Add a new protocol on jdoe.test.org that uses gsiftp",
             "$ rucio rse protocol remove --scheme gsiftp --rse JDOE_DATADISK # Remove the existing gsiftp protocol",
         ]
 

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/client/commands/rule.py

@@ -51,7 +51,7 @@ class Rule(CommandBase):
         parser.add_argument("-a", "--account", dest="rule_account", action="store", help="The account of the rule")
         parser.add_argument("--activity", action="store", help="Activity to be used (e.g. User, Data Consolidation)")  # TODO More info on this
         parser.add_argument("--rule-id", help="The rule ID, for accessing an existing rule.")
-        parser.add_argument("--lifetime", dest="lifetime", action="store", type=int, help="Rule lifetime (in seconds)")
+        parser.add_argument("--lifetime", dest="lifetime", type=str, help="Rule lifetime (in seconds). Use 'None' for no set lifetime")
         parser.add_argument("--locked", action="store_true", help="Set the rule to locked")  # TODO More info on this
         parser.add_argument("--source-replica-expression", help="RSE Expression for RSEs to be considered for source replicas")
         parser.add_argument("--comment", dest="comment", action="store", help="Comment about the replication rule")

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/client/richclient.py

@@ -15,7 +15,7 @@
 import logging
 import pydoc
 import re
-import subprocess
+import shutil
 import sys
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Optional, Union
@@ -309,16 +309,9 @@ def get_pager() -> 'Callable[[str], None]':
     :returns: pager
     """
     default_pager = 'less'
-    try:
-        result = subprocess.run([default_pager], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        less_available = result.returncode == 0
-    except Exception:
-        less_available = False
-
-    def less_pager_function(text: str) -> None:
-        """Use the 'less' pager with -FRSXKM options."""
-        pydoc.pipepager(text, f'{default_pager} -FRSXKM')
-
-    if less_available:
-        return less_pager_function
+    # Attempt to use the default pager if available.
+    if shutil.which(default_pager) is not None:
+        return lambda text: pydoc.pipepager(text, f'{default_pager} -FRSXKM')
+
+    # Fall back to pydoc.pager if the default pager is not available.
     return pydoc.pager

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/common/config.py

@@ -547,12 +547,12 @@ def config_get_list(
         section: str,
         option: str,
         *,
-        default: list[str] = ...,
+        default: _T = ...,
         check_config_table: bool = ...,
         session: "Optional[Session]" = ...,
         use_cache: bool = ...,
         expiration_time: int = ...,
-) -> list[str]:
+) -> Union[list[str], _T]:
     ...
 
 

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/rse/protocols/gfal.py

@@ -186,6 +186,7 @@ class Default(protocol.RSEProtocol):
         self.__ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"])
         self.__ctx.set_opt_string("XROOTD PLUGIN", "XRD.WANTPROT", "gsi,unix")
         self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False)
+        self.__ctx.set_opt_boolean("HTTP PLUGIN", "RETRIEVE_BEARER_TOKEN", False)
         auth_configured = False
         if self.auth_token:
             self.__ctx.set_opt_string("BEARER", "TOKEN", self.auth_token)
@@ -488,12 +489,12 @@ class Default(protocol.RSEProtocol):
 
         try:
             for path in paths:
-                if self.__gfal2_exist(path) == 0:
-                    ret = ctx.unlink(str(path))
-                    if ret:
-                        return ret
-                else:
-                    raise exception.SourceNotFound
+                # GFAL does a PROPFIND request before DELETE when the scheme is
+                # davs://, which is wasteful.
+                path = re.sub('^davs://', 'https://', str(path))
+                ret = ctx.unlink(path)
+                if ret:
+                    return ret
             return ret
         except gfal2.GError as error:  # pylint: disable=no-member
             if error.code == errno.ENOENT or 'No such file' in str(error):

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/rse/protocols/webdav.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 import sys
 from dataclasses import dataclass
@@ -161,6 +162,7 @@ class Default(protocol.RSEProtocol):
             :raises RSEAccessDenied
         """
         credentials = credentials or {}
+        using_presigned_urls = self.rse['sign_url'] is not None
         try:
             parse_url = urlparse(self.path2pfn(''))
             self.server = f'{parse_url.scheme}://{parse_url.netloc}'
@@ -177,23 +179,29 @@ class Default(protocol.RSEProtocol):
         except KeyError:
             self.auth_type = 'cert'
 
-        try:
-            self.cert = credentials['cert']
-        except KeyError:
-            x509 = os.getenv('X509_USER_PROXY')
-            if not x509:
-                # Trying to get the proxy from the default location
-                proxy_path = '/tmp/x509up_u%s' % os.geteuid()
-                if os.path.isfile(proxy_path):
-                    self.cert = (proxy_path, proxy_path)
-                elif self.auth_token:
-                    # If no proxy is found, we set the cert to None and use the auth_token
-                    self.cert = None
-                    pass
+        if using_presigned_urls:
+            # Suppress all authentication, otherwise S3 servers will reject
+            # requests.
+            self.cert = None
+            self.auth_token = None
+        else:
+            try:
+                self.cert = credentials['cert']
+            except KeyError:
+                x509 = os.getenv('X509_USER_PROXY')
+                if not x509:
+                    # Trying to get the proxy from the default location
+                    proxy_path = '/tmp/x509up_u%s' % os.geteuid()
+                    if os.path.isfile(proxy_path):
+                        self.cert = (proxy_path, proxy_path)
+                    elif self.auth_token:
+                        # If no proxy is found, we set the cert to None and use the auth_token
+                        self.cert = None
+                        pass
+                    else:
+                        raise exception.RSEAccessDenied('X509_USER_PROXY is not set')
                 else:
-                    raise exception.RSEAccessDenied('X509_USER_PROXY is not set')
-            else:
-                self.cert = (x509, x509)
+                    self.cert = (x509, x509)
 
         try:
             self.timeout = credentials['timeout']
@@ -205,11 +213,16 @@ class Default(protocol.RSEProtocol):
             self.session.headers.update({'Authorization': 'Bearer ' + self.auth_token})
         # "ping" to see if the server is available
         try:
-            res = self.session.request('HEAD', self.path2pfn(''), verify=False, timeout=self.timeout, cert=self.cert)
-            if res.status_code != 200:
-                raise exception.ServiceUnavailable('Problem to connect %s : %s' % (self.path2pfn(''), res.text))
+            test_url = self.path2pfn('')
+            res = self.session.request('HEAD', test_url, verify=False, timeout=self.timeout, cert=self.cert)
+            # REVISIT: this test checks some URL that doesn't correspond to
+            # any valid Rucio file.  Although this works for normal WebDAV
+            # endpoints, it fails for endpoints using presigned URLs.  As a
+            # work-around, accept 4xx status codes when using presigned URLs.
+            if res.status_code != 200 and not (using_presigned_urls and res.status_code < 500):
+                raise exception.ServiceUnavailable('Bad status code %s %s : %s' % (res.status_code, test_url, res.text))
         except requests.exceptions.ConnectionError as error:
-            raise exception.ServiceUnavailable('Problem to connect %s : %s' % (self.path2pfn(''), error))
+            raise exception.ServiceUnavailable('Problem to connect %s : %s' % (test_url, error))
         except requests.exceptions.ReadTimeout as error:
             raise exception.ServiceUnavailable(error)
 
@@ -552,3 +565,30 @@ class Default(protocol.RSEProtocol):
             return totalsize, unusedsize
         except Exception as error:
             raise exception.ServiceUnavailable(error)
+
+
+class NoRename(Default):
+    """ Implementing access to RSEs using the WebDAV protocol but without
+    renaming files on upload/download. Necessary for some storage endpoints.
+    """
+
+    def __init__(self, protocol_attr, rse_settings, logger=logging.log):
+        """ Initializes the object with information about the referred RSE.
+
+            :param protocol_attr:  Properties of the requested protocol.
+            :param rse_settings:   The RSE settings.
+            :param logger:         Optional decorated logger that can be passed from the calling daemons or servers.
+        """
+        super(NoRename, self).__init__(protocol_attr, rse_settings, logger=logger)
+        self.renaming = False
+        self.attributes.pop('determinism_type', None)
+
+    def rename(self, pfn, new_pfn):
+        """ Allows to rename a file stored inside the connected RSE.
+
+            :param pfn:      Current physical file name
+            :param new_pfn  New physical file name
+
+            :raises DestinationNotAccessible, ServiceUnavailable, SourceNotFound
+        """
+        raise NotImplementedError

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/lib/rucio/vcsversion.py

@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '36.3.0',
+    'version': '36.5.0',
     'branch_nick': 'release-36',
-    'revision_id': '681dff60333b2f0b7d3f0b1eb7bf42ccc5f15053',
-    'revno': 13473
+    'revision_id': 'd435b5bdf4f37f698d65d0a1ef4a7ecd21305b35',
+    'revno': 13494
 }

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/tests/test_bin_rucio.py

@@ -123,6 +123,28 @@ class TestBinRucio:
         print(out, )
         assert 'Added new account: %s\n' % tmp_val in out
 
+    @pytest.mark.dirty
+    def test_list_account(self):
+        """CLIENT(ADMIN): List accounts"""
+        n_accounts = 5
+        tmp_accounts = [account_name_generator() for _ in range(n_accounts)]
+        for account in tmp_accounts:
+            execute(f'rucio-admin account add {account}')
+            execute(f'rucio-admin account add-attribute {account} --key test_list_account --value true')
+
+        cmd = 'rucio-admin account list'
+        exitcode, out, err = execute(cmd)
+        assert tmp_accounts[0] in out
+        assert tmp_accounts[-1] in out  # Test by induction
+
+        cmd = "rucio-admin account list --filter test_list_account=true"
+        exitcode, out, err = execute(cmd)
+        assert set([o for o in out.split("\n") if o != '']) == set(tmp_accounts)  # There's a little '' printed after
+
+        cmd = "rucio-admin account list --filter test_list_account=true --csv"
+        exitcode, out, err = execute(cmd)
+        assert set(o.rstrip('\n') for o in out.split(',')) == set(tmp_accounts)  # Last obj in list has a `\n` included
+
     def test_whoami(self):
         """CLIENT(USER): Rucio whoami"""
         cmd = 'rucio whoami'
@@ -224,7 +246,8 @@ class TestBinRucio:
         assert 'Added new non-deterministic RSE: %s\n' % tmp_val in out
 
     def test_list_rses(self):
-        """CLIENT(ADMIN): List RSEs"""
+        """CLIENT(USER/ADMIN): List RSEs"""
+        # TODO Test filter
         tmp_val = rse_name_generator()
         cmd = 'rucio-admin rse add %s' % tmp_val
         exitcode, out, err = execute(cmd)
@@ -234,6 +257,19 @@ class TestBinRucio:
         print(out, )
         assert tmp_val in out
 
+        # Expected output is a new RSE on each line
+        cmd = 'rucio-admin rse list --csv'
+        _, out, _ = execute(cmd)
+        assert tmp_val in out.split('\n')
+
+        cmd = 'rucio list-rses'
+        _, out, _ = execute(cmd)
+        assert tmp_val in out
+
+        cmd = 'rucio list-rses --csv'
+        _, out, _ = execute(cmd)
+        assert tmp_val in out.split('\n')
+
     def test_rse_add_distance(self):
         """CLIENT (ADMIN): Add distance to RSE"""
         # add RSEs

{rucio_clients-36.3.0 → rucio_clients-36.5.0}/tests/test_cli_client_structure.py

@@ -21,6 +21,22 @@ from rucio.common.utils import generate_uuid
 from rucio.tests.common import account_name_generator, execute, file_generator, rse_name_generator, scope_name_generator
 
 
+def test_main_args():
+    specify_account = "rucio --account root --auth-strategy userpass whoami"
+    exitcode, out, err = execute(specify_account)
+
+    assert exitcode == 0
+    assert "This method is being deprecated" not in err
+    assert "root" in out
+
+    legacy_arg = "rucio --legacy --account root --auth-strategy userpass whoami"
+    exitcode, out, err = execute(legacy_arg)
+
+    assert exitcode == 0
+    assert "This method is being deprecated" in err
+    assert "root" in out
+
+
 def test_account(rucio_client):
     new_account = account_name_generator()
     command = f"rucio account add --type USER --account {new_account}"
@@ -613,6 +629,20 @@ def test_rule(rucio_client, mock_scope):
     assert exitcode == 0
     assert "ERROR" not in err
 
+    # Testing the two different lifetime type options
+    cmd = f"rucio rule update --rule-id {rule_id} --lifetime 10"
+    exitcode, out, err = execute(cmd)
+    assert exitcode == 0
+    assert "ERROR" not in err
+    rule_info = rucio_client.get_replication_rule(rule_id)
+    assert abs(rule_info["updated_at"] - rule_info["expires_at"]).seconds == 10
+
+    cmd = f"rucio rule update --rule-id {rule_id} --lifetime None"
+    exitcode, out, err = execute(cmd)
+    assert exitcode == 0
+    assert "ERROR" not in err
+    assert rucio_client.get_replication_rule(rule_id)["expires_at"] is None
+
     # Do one without a child rule so i can delete it
     new_name = generate_uuid()
     rucio_client.add_replica(mock_rse, scope, new_name, 4, "deadbeef")