rucio-clients 35.6.0__tar.gz → 36.0.0.post1__tar.gz

{rucio_clients-35.6.0 → rucio_clients-36.0.0.post1}/AUTHORS.rst

@@ -67,6 +67,7 @@ Individual contributors to the source code
 - Aksel Lunde Aase <aksel.lunde.aase@gmail.com>, 2022
 - Elena Gazzarrini <gazzarrini.elena@gmail.com>, 2022-2023
 - Maximilian Linhoff, <maximilian.linhoff@tu-dortmund.de>, 2024
+- Eric Banzuzi, <eric.banzuzi@gmail.com>, 2024
 
 Organisations employing contributors
 ------------------------------------

{rucio_clients-35.6.0 → rucio_clients-36.0.0.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rucio-clients
-Version: 35.6.0
+Version: 36.0.0.post1
 Summary: Rucio Client Lite Package
 Home-page: https://rucio.cern.ch/
 Author: Rucio
@@ -24,6 +24,8 @@ Requires-Dist: urllib3>=1.26.18
 Requires-Dist: dogpile-cache>=1.2.2
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: jsonschema>=4.20.0
+Requires-Dist: packaging>=24.1
+Requires-Dist: rich>=13.7.1
 Provides-Extra: ssh
 Requires-Dist: paramiko>=3.4.0; extra == "ssh"
 Provides-Extra: kerberos

rucio_clients-36.0.0.post1/bin/rucio

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+from typing import Optional
+
+from rucio.client.commands.bin_legacy.rucio import main as main_legacy
+from rucio.client.commands.command import main
+
+
+def make_warning():
+    base_warning = "\nWARNING: This method is being deprecated."
+    new_command = map_legacy_command()
+    if new_command is not None:
+        warning = f"{base_warning}\nPlease replace your command with `rucio {' '.join(new_command)}`.\n"
+    else:
+        warning = base_warning + "\nPlease view rucio -h for an updated help menu.\n"
+
+    return warning
+
+
+def map_legacy_command() -> Optional[list[str]]:
+    try:
+        command = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--version")][1]
+    except IndexError:
+        command = None
+
+    new_command = None
+    if command not in ("-h", "--version", None):
+        command_map = {
+            "list-file-replicas": ["replica", "list"],
+            "list-dataset-replicas": ["replica", "dataset"],
+            "add-dataset": ["did", "add", "--type dataset"],
+            "add-container": ["did", "add", "--type container"],
+            "attach": ["did", "content", "add"],
+            "detach": ["did", "content", "remove"],
+            "ls": ["did", "list"],
+            "list-dids": ["did", "list"],
+            "list-parent-dids": ["did", "show", "--parent"],
+            "list-scopes": ["scope", "list"],
+            "close": ["did", "update", "--close"],
+            "reopen": ["did", "update", "--open"],
+            "stat": ["did", "show"],
+            "erase": ["did", "remove"],
+            "list-content": ["did", "content", "list"],
+            "list-content-history": ["did", "content", "history"],
+            "upload": ["upload"],
+            "get": ["download"],
+            "download": ["download"],
+            "get-metadata": ["did", "metadata", "list"],
+            "set-metadata": ["did", "metadata", "add"],
+            "delete-metadata": ["did", "metadata", "remove"],
+            "list-rse-usage": ["rse", "show"],
+            "list-account-usage": ["account", "usage"],
+            "list-account-limits": ["account", "limit", "list"],
+            "add-rule": ["rule", "add"],
+            "delete-rule": ["rule", "remove"],
+            "rule-info": ["rule", "info"],
+            "list-rules": ["rule", "list"],
+            "list-rules-history": ["rule", "history"],
+            "update-rule": ["rule", "update"],
+            "move-rule": ["rule", "update", "--move"],
+            "list-rses": ["rse", "list"],
+            "list-suspicious-replicas": ["replica", "state", "suspicious"],
+            "list-rse-attributes": ["rse", "attribute", "list"],
+            "touch": ["did", "update", "--touch"],
+            "add-lifetime-exception": ["lifetime-exception", "add"],
+        }
+        try:
+            new_command = command_map[command]
+        except KeyError:
+            pass
+
+    return new_command
+
+
+if __name__ == "__main__":
+    commands = ("-h", "--help", "--version", "account", "config", "did", "replica", "rse", "rule", "scope", "subscription", "ping", "whoami", "test-server", "lifetime-exception", "upload", "download")
+    first_command = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--help", "--version")][1]
+    is_legacy = '--legacy' in sys.argv
+
+    if (first_command in commands) and not is_legacy:
+        main()
+
+    elif is_legacy:
+        warning = make_warning()
+        print(warning)
+        sys.argv.pop(sys.argv.index('--legacy'))
+        main_legacy()
+
+    else:
+        warning = make_warning()
+        print(warning)
+        try:
+            main_legacy()
+        # Make a custom warning - show the new help menu when invalid commands are called.
+        except argparse.ArgumentError:
+            print("Invalid argument(s) - %s " % sys.argv[1:])
+            command = map_legacy_command()
+            if command is not None:
+                sys.argv = ["rucio"] + command + ["-h"]
+            else:
+                sys.argv = ["rucio", "-h"]
+
+            main()

rucio_clients-36.0.0.post1/bin/rucio-admin

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+from rucio.client.commands.bin_legacy.rucio_admin import main as main_legacy
+
+
+def make_warning():
+    base_warning = "\nWARNING: This method is being deprecated."
+    args = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--help", "--version")]
+    first_command = args[1]
+    try:
+        second_command = args[2]
+    except IndexError:
+        second_command = "-h"
+
+    if first_command not in ("-h", "--version", "--help"):
+        command_map = {
+            "data": {"-h": "did -h", "import": "did upload", "export": "did download"},
+            "account": {
+                "-h": "account -h",
+                "add": "account add",
+                "delete": "account remove",
+                "info": "account show",
+                "update": "account update",
+                "set-limits": "account limit add",
+                "get-limits": "account limit list",
+                "delete-limits": "account limit remove",
+                "ban": "account update --ban True",
+                "unban": "account update --ban False",
+                "list-attributes": "account attribute list",
+                "add-attribute": "account attribute add",
+                "delete-attribute": "account attribute remove",
+            },
+            "identity": {"-h": "account identity -h", "add": "account identity add", "delete": "account identity remove"},
+            "rse": {
+                "-h": "rse -h",
+                "add": "rse add",
+                "list": "rse",
+                "update": "rse update",
+                "info": "rse info",
+                "set-attribute": "rse attribute add",
+                "delete-attribute": "rse attribute remove",
+                "delete-distance": "rse distance remove",
+                "get-distance": "rse distance list",
+                "get-attribute": "rse attribute list",
+                "add-protocol": "rse protocol add",
+                "delete-protocol": "rse protocol remove",
+                "delete": "rse remove",
+                "add-qos-policy": "rse qos add",
+                "add-distance": "rse distance add",
+                "delete-qos-policy": "rse qos remove",
+                "list-qos-policies": "rse qos list",
+                "set-limit": "rse limit add",
+                "delete-limit": "rse limit remove",
+            },
+            "scope": {"-h": "scope -h", "add": "scope add", "list": "scope list"},
+            "config": {"-h": "config -h", "get": "config list", "set": "config add", "delete": "config remove"},
+            "subscription": {"-h": "subscription -h", "add": "subscription add", "list": "subscription", "update": "subscription update", "reevaluate": "subscription touch"},
+            "replicas": {"-h": "replica -h", "quarantine": "replica state quarantine", "declare-bad": "replica state bad", "declare-temporary-unavailable": "replica state temp-unavailable", "list-pfns": "replica pfn", "set-tombstone": "replica tombstone"},
+        }
+        try:
+            new_command = command_map[first_command]
+            new_command = new_command[second_command]
+        except KeyError:
+            new_command = "-h"
+
+        warning = f"{base_warning}\nPlease replace your command with `rucio {new_command}`.\n"
+    else:
+        warning = base_warning + "\nPlease view rucio -h for an updated help menu.\n"
+
+    print(warning)
+
+
+if __name__ == "__main__":
+    make_warning()
+    main_legacy()

{rucio_clients-35.6.0 → rucio_clients-36.0.0.post1}/etc/rucio.cfg.atlas.client.template

@@ -29,8 +29,9 @@ request_retries = 3
 auth_type = x509_proxy
 
 [policy]
-permission = atlas
-schema = atlas
+package = atlas_rucio_policy_package
+#permission = atlas
+#schema = atlas
 lfn2pfn_algorithm_default = hash
 support = hn-atlas-dist-analysis-help@cern.ch
 support_rucio = https://github.com/rucio/rucio/issues/

{rucio_clients-35.6.0 → rucio_clients-36.0.0.post1}/etc/rucio.cfg.template

@@ -229,9 +229,10 @@ panda_url = http://bigpanda.cern.ch/jobs/?category=analysis&jobstatus=running
 window = 604800
 
 [policy]
-package = ATLASRucioPolicy
+package = atlas_rucio_policy_package
 permission = atlas
 schema = atlas
+extract_scope = atlas
 lfn2pfn_algorithm_default = hash
 support = hn-atlas-dist-analysis-help@cern.ch
 support_rucio = https://github.com/rucio/rucio/issues/

{rucio_clients-35.6.0 → rucio_clients-36.0.0.post1}/lib/rucio/client/baseclient.py

@@ -22,13 +22,11 @@ import os
 import secrets
 import sys
 import time
-from collections.abc import Generator
 from configparser import NoOptionError, NoSectionError
-from logging import Logger
 from os import environ, fdopen, geteuid, makedirs, path
 from shutil import move
 from tempfile import mkstemp
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
 
 import requests
@@ -44,6 +42,10 @@ from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupporte
 from rucio.common.extra import import_extras
 from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from logging import Logger
+
 EXTRA_MODULES = import_extras(['requests_kerberos'])
 
 if EXTRA_MODULES['requests_kerberos']:
@@ -90,7 +92,7 @@ class BaseClient:
                  timeout: Optional[int] = 600,
                  user_agent: Optional[str] = 'rucio-clients',
                  vo: Optional[str] = None,
-                 logger: Logger = LOG) -> None:
+                 logger: 'Logger' = LOG) -> None:
         """
         Constructor of the BaseClient.
         :param rucio_host: The address of the rucio server, if None it is read from the config file.
@@ -364,7 +366,7 @@ class BaseClient:
         else:
             return exception.RucioException, "%s: %s" % (exc_cls, exc_msg)
 
-    def _load_json_data(self, response: requests.Response) -> Generator[Any, Any, Any]:
+    def _load_json_data(self, response: requests.Response) -> 'Generator[Any, Any, Any]':
         """
         Helper method to correctly load json data based on the content type of the http response.
 

rucio_clients-36.0.0.post1/lib/rucio/client/commands/__init__.py

@@ -0,0 +1,25 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from rucio.client.commands.account import Account  # noqa: F401
+from rucio.client.commands.command_base import CommandBase  # noqa: F401
+from rucio.client.commands.config import Config  # noqa: F401
+from rucio.client.commands.did import DID  # noqa: F401
+from rucio.client.commands.download import Download  # noqa: F401
+from rucio.client.commands.lifetime_exception import LifetimeException  # noqa: F401
+from rucio.client.commands.replica import Replica  # noqa: F401
+from rucio.client.commands.rse import RSE  # noqa: F401
+from rucio.client.commands.rule import Rule  # noqa: F401
+from rucio.client.commands.scope import Scope  # noqa: F401
+from rucio.client.commands.subscription import Subscription  # noqa: F401
+from rucio.client.commands.upload import Upload  # noqa: F401

rucio_clients-36.0.0.post1/lib/rucio/client/commands/account.py

@@ -0,0 +1,219 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from argparse import SUPPRESS
+from typing import TYPE_CHECKING
+
+from rucio.client.commands.bin_legacy.rucio import list_account_usage
+from rucio.client.commands.bin_legacy.rucio_admin import (
+    add_account,
+    add_account_attribute,
+    ban_account,
+    delete_account,
+    delete_account_attribute,
+    delete_limits,
+    identity_add,
+    identity_delete,
+    info_account,
+    list_account_attributes,
+    list_accounts,
+    list_identities,
+    set_limits,
+    unban_account,
+    update_account,
+)
+from rucio.client.commands.command_base import CommandBase
+
+if TYPE_CHECKING:
+    from argparse import ArgumentParser
+
+    from rucio.client.commands.utils import OperationDict
+
+
+class Account(CommandBase):
+    def module_help(self) -> str:
+        return "Methods to add or change accounts for users, groups, and services. Used to assign privileges."
+
+    def usage_example(self) -> list[str]:
+        return [
+            "$ rucio account list # List all accounts on the instance",
+            "$ rucio account add --account user_jdoe --type USER  # Create a new user account",
+            "$ rucio account update --account user_jdoe --email jdoe@cern.ch  # Update jdoe's email",
+            "$ rucio account update --account jdoe --ban True  # Ban jdoe",
+            "$ rucio account history --account root  # Show all the usage history for the account root",
+        ]
+
+    def list_namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("--type", dest="account_type", help="Account Type", choices={"USER", "GROUP", "SERVICE"})
+        parser.add_argument("-a", "--account", dest="account", help="Account name")
+        parser.add_argument("--id", dest="identity", action="store", help="Identity (e.g. DN)")
+        parser.add_argument("--filters", dest="filters", action="store", help="Filter arguments in form `key=value,another_key=next_value`")
+
+    def add_namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("--type", dest="accounttype", help="Account Type (USER, GROUP, SERVICE)", required=True)
+        parser.add_argument("-a", "--account", dest="account", help="Account name", required=True)
+        parser.add_argument("--email", dest="accountemail", help="Add an email address associated with the account")
+
+    def show_namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("-a", "--account", dest="account", help="Account name", required=True)
+
+    def update_namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("-a", "--account", help="Account name", required=True)
+        parser.add_argument("--email", help="Account email")
+        parser.add_argument("--ban", type=bool, choices=(True, False), help='Ban the account, to disable it. Use --ban False to unban.', default=None)
+
+    def remove_namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("-a", "--account", dest="acnt", action="store", help="Account name", required=True)
+
+    def _operations(self) -> dict[str, "OperationDict"]:
+        return {
+            "list": {"call": self.list_, "docs": "List accounts", "namespace": self.list_namespace},
+            "add": {"call": self.add, "docs": "Add a new account", "namespace": self.add_namespace},
+            "show": {"call": self.show, "docs": "Get all stats on an account, including status, account type, and dates of creation and updates", "namespace": self.show_namespace},
+            "remove": {"call": self.remove, "docs": "Delete an account", "namespace": self.remove_namespace},
+            "update": {"call": self.update, "docs": "Change the basic account settings", "namespace": self.update_namespace},
+        }
+
+    def implemented_subcommands(self) -> dict[str, type[CommandBase]]:
+        return {"attribute": Attribute, "limit": Limit, "identity": Identity}
+
+    def list_(self):
+        list_accounts(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def add(self):
+        add_account(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def remove(self):
+        delete_account(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def show(self):
+        info_account(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def update(self):
+        if self.args.ban is not None:
+            if self.args.ban:
+                ban_account(self.args, self.client, self.logger, self.console, self.spinner)
+            else:
+                unban_account(self.args, self.client, self.logger, self.console, self.spinner)
+        else:
+            # Temp fix to allow specific kwargs
+            self.args.key = "email"
+            self.args.value = self.args.email
+            update_account(self.args, self.client, self.logger, self.console, self.spinner)
+
+
+class Attribute(Account):
+    def module_help(self) -> str:
+        return "Add additional key/value pairs associated with an account."
+
+    def implemented_subcommands(self) -> dict[str, type[CommandBase]]:
+        return {}
+
+    def namespace(self, subparser):
+        subparser.add_argument("-a", "--account", help="Account name")
+        subparser.add_argument("--key", dest="key", action="store", help="Attribute key")
+        subparser.add_argument("--value", dest="value", action="store", help="Attribute value")
+
+    def usage_example(self) -> list[str]:
+        return ["$ rucio account attribute list --account jdoe  # Show all attributes for jdoe",
+                "$ rucio -v account attribute add --account jdoe --key test_key --value true  # Set test_key = true for jdoe"]
+
+    def _operations(self) -> dict[str, "OperationDict"]:
+        return {
+            "list": {"call": self.list_, "docs": "List all account attributes"},
+            "add": {"call": self.add, "docs": "Add a new attribute to an account or update an existing one"},
+            "remove": {"call": self.remove, "docs": "Remove an existing account attribute"},
+        }
+
+    def list_(self):
+        list_account_attributes(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def add(self):
+        add_account_attribute(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def remove(self):
+        delete_account_attribute(self.args, self.client, self.logger, self.console, self.spinner)
+
+
+class Limit(Account):
+    def module_help(self) -> str:
+        return "Manage storage limits for an account at a given RSE."
+
+    def namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("-a", "--account", dest="account", help="Account name", required=True)
+        parser.add_argument("--rses", "--rse-exp", dest='rse', action="store", help="RSE expression")
+        parser.add_argument("--bytes", action="store", help='Value can be specified in bytes ("10000"), with a storage unit ("10GB"), or "infinity"')
+        parser.add_argument("--locality", nargs="?", default="local", choices=["local", "global"], help="Global or local limit scope")
+        parser.add_argument("--human", default=True, help=SUPPRESS)
+
+    def _operations(self) -> dict[str, "OperationDict"]:
+        return {
+            "list": {"call": self.list_, "docs": "Show limits and current utilization for an account at a given RSE", "namespace": self.namespace},
+            "add": {"call": self.add, "docs": "Add or update limits for an account", "namespace": self.namespace},  # TODO Add and update should become different operations
+            "remove": {"call": self.remove, "docs": "Remove all limits for given account/rse/locality", "namespace": self.namespace},
+        }
+
+    def implemented_subcommands(self) -> dict[str, type[CommandBase]]:
+        return {}
+
+    def usage_example(self) -> list[str]:
+        return super().usage_example()
+
+    def list_(self):
+        self.args.usage_account = self.args.account
+        list_account_usage(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def add(self):
+        set_limits(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def remove(self):
+        delete_limits(self.args, self.client, self.logger, self.console, self.spinner)
+
+
+class Identity(Account):
+    def module_help(self) -> str:
+        return "Manage identities on an account."
+
+    def namespace(self, parser: "ArgumentParser") -> None:
+        parser.add_argument("--account", dest="account", action="store", help="Account name", required=True)
+        parser.add_argument("--type", dest="authtype", action="store", choices=["X509", "GSS", "USERPASS", "SSH", "SAML", "OIDC"], help="Authentication type")
+        parser.add_argument("--id", dest="identity", action="store", help="Identity as a DNs for X509 IDs.")
+        parser.add_argument("--email", dest="email", action="store", help="Email address associated with the identity")
+        parser.add_argument("--password", dest="password", action="store", help="Password if authtype is USERPASS")
+        parser.add_argument("--human", default=True, help=SUPPRESS)
+
+    def _operations(self) -> dict[str, "OperationDict"]:
+        return {
+            "list": {"call": self.list_, "docs": "Display existing identities for an account", "namespace": self.namespace},
+            "add": {"call": self.add, "docs": "Grant identity access to an account", "namespace": self.namespace},
+            "remove": {"call": self.remove, "docs": "Revoke identity access for an account", "namespace": self.namespace},
+        }
+
+    def usage_example(self) -> list[str]:
+        return [
+            "$ rucio account identity list --account jdoe  # List all auth identities for jdoe",
+            "$ rucio account identity add --account jdoe --type GSS --email jdoe@cern.ch --id jdoe@fnal.ch  # Add a new GSS auth",
+            "$ rucio account identity add --account jdoe --type X509 --id 'CN=Joe Doe,CN=707658,CN=jdoe,OU=Users,OU=Organic Units,DC=cern,DC=ch' --email jdoe@cern.ch  # Add a new X509 auth",
+        ]
+
+    def implemented_subcommands(self) -> dict[str, type[CommandBase]]:
+        return {}
+
+    def list_(self):
+        list_identities(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def add(self):
+        identity_add(self.args, self.client, self.logger, self.console, self.spinner)
+
+    def remove(self):
+        identity_delete(self.args, self.client, self.logger, self.console, self.spinner)