rucio-clients 37.0.0rc1__py3-none-any.whl

rucio/cli/scope.py

@@ -0,0 +1,40 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import click
+
+from rucio.cli.bin_legacy.rucio_admin import add_scope, list_scopes
+from rucio.cli.utils import Arguments
+
+
+@click.group()
+def scope():
+    """Interact with scopes - a logical grouping of DIDs"""
+
+
+@scope.command("add")
+@click.argument("scope-name")
+@click.option("-a", "--account", help="Associated account", required=True)
+@click.pass_context
+def add_(ctx, account, scope_name):
+    """Add a new scope with name [SCOPE-NAME]"""
+    args = Arguments({"scope": scope_name, "account": account})
+    add_scope(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
+
+
+@scope.command("list")
+@click.option("-a", "--account", help="Filter by associated account", required=False)
+@click.pass_context
+def list_(ctx, account):
+    """List existing scopes"""
+    list_scopes(Arguments({"account": account}), ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)

rucio/cli/subscription.py

@@ -0,0 +1,73 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import click
+
+from rucio.cli.bin_legacy.rucio_admin import add_subscription, list_subscriptions, reevaluate_did_for_subscription, update_subscription
+from rucio.cli.utils import Arguments
+
+
+@click.group()
+def subscription():
+    "The methods for automated and regular processing of some specific rules"
+
+
+@subscription.command("show")
+@click.option("-a", "--account", help="Account associated with the subscription")
+@click.option("--long", default=False, is_flag=True, help="Show extended information about the subscription")
+@click.argument("subscription-name")
+@click.pass_context
+def list_(ctx, subscription_name, account, long):
+    """Show the attributes of a subscription [SUBSCRIPTION-NAME]"""
+    args = Arguments({"subs_account": account, "name": subscription_name, "long": long})
+    list_subscriptions(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
+
+
+@subscription.command("update")
+@click.argument("subscription-name")
+@click.option("--filter", "did_filter", help='Json serializable DID filter (eg \'{"scope": ["tests"], "project": ["data12_8TeV"]}\')', required=True)
+@click.option("--rule", help='List of replication rules (eg \'[{"activity": "Functional Tests", "copies": 2, "rse_expression": "tier=2", "lifetime": 3600, "weight": "mou"}]\')', required=True)
+@click.option("--comment", help="Comments on subscription")
+@click.option("--lifetime", type=int, help="Subscription lifetime (in days)")
+@click.option("--account", help="Account name")
+@click.option("--priority", help="The priority of the subscription")
+@click.pass_context
+def update(ctx, subscription_name, did_filter, rule, comment, lifetime, account, priority):
+    """Update a subscription [SUBSCRIPTION-NAME] to have new properties"""
+    args = Arguments({"name": subscription_name, "filter": did_filter, "replication_rules": rule, "comments": comment, "lifetime": lifetime, "subs_account": account, "priority": priority})
+    update_subscription(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
+
+
+@subscription.command("add")
+@click.argument("subscription-name")
+@click.option("--filter", "did_filter", help='Json serializable DID filter (eg \'{"scope": ["tests"], "project": ["data12_8TeV"]}\')', required=True)
+@click.option("--rule", help='List of replication rules (eg \'[{"activity": "Functional Tests", "copies": 2, "rse_expression": "tier=2", "lifetime": 3600, "weight": "mou"}]\')', required=True)
+@click.option("--comment", help="Comments on subscription")
+@click.option("--lifetime", type=int, help="Subscription lifetime (in days)")
+@click.option("--account", help="Account name")
+@click.option("--priority", help="The priority of the subscription")
+@click.pass_context
+def add_(ctx, subscription_name, did_filter, rule, comment, lifetime, account, priority):
+    """Create a new subscription with the name [SUBSCRIPTION-NAME]"""
+    args = Arguments({"name": subscription_name, "filter": did_filter, "replication_rules": rule, "comments": comment, "lifetime": lifetime, "subs_account": account, "priority": priority})
+    add_subscription(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
+
+
+@subscription.command("touch")
+@click.argument("dids", nargs=-1)
+@click.pass_context
+def touch(ctx, dids):
+    """Reevaluate list of DIDs against all active subscriptions"""
+    # TODO make reeval accept dids as a list
+    dids = ",".join(dids)
+    reevaluate_did_for_subscription(Arguments({"dids": dids}), ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)

rucio/cli/upload.py

@@ -0,0 +1,60 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import click
+
+from rucio.cli.bin_legacy.rucio import upload
+from rucio.cli.utils import Arguments
+from rucio.common.config import config_get_float
+
+
+@click.command("upload")
+@click.argument("file-paths", nargs=-1)
+@click.option("--rse", "--rse-name", help="Rucio Storage Element (RSE) name", required=True)
+@click.option("--lifetime", type=int, help="Lifetime of the rule in seconds")
+@click.option("--expiration-date", help="The date when the rule expires in UTC, format: <year>-<month>-<day>-<hour>:<minute>:<second>. E.g. 2022-10-20-20:00:00")
+@click.option("--scope", help="Scope name.")
+@click.option("--impl", type=click.Choice([]), help="Transfer protocol implementation to use (e.g: xrootd, gfal.NoRename, webdav, ssh.Rsync, rclone)")
+# The --no-register option is hidden. This is pilot ONLY. Users should not use this. Will lead to unregistered data on storage!
+@click.option("--no-register", is_flag=True, default=False, hidden=True)
+@click.option("--register-after-upload", is_flag=True, default=False, help="Register the file _only_ after successful upload")
+@click.option("--summary", is_flag=True, default=False, help="Create rucio_upload.json summary file")
+@click.option("--guid", help="Manually specify the GUID for the file.")
+@click.option("--protocol", help="Force the protocol to use")
+@click.option("--pfn", help="Specify the exact PFN for the upload")
+@click.option("--lfn", help="Specify the exact LFN for the upload")
+@click.option("--transfer-timeout", type=float, default=config_get_float("upload", "transfer_timeout", False, 360), help="Transfer timeout (in seconds)")
+@click.option("-r", "--recursive", is_flag=True, default=False, help="Convert recursively the folder structure into collections")
+@click.pass_context
+def upload_command(ctx, file_paths, rse, lifetime, expiration_date, scope, impl, no_register, register_after_upload, summary, guid, protocol, pfn, lfn, transfer_timeout, recursive):
+    """Upload file(s) to a Rucio RSE"""
+    args = Arguments(
+        {
+            "args": file_paths,
+            "rse": rse,
+            "lifetime": lifetime,
+            "expiration_date": expiration_date,
+            "scope": scope,
+            "impl": impl,
+            "no_register": no_register,
+            "register_after_upload": register_after_upload,
+            "protocol": protocol,
+            "summary": summary,
+            "guid": guid,
+            "pfn": pfn,
+            "name": lfn,
+            "transfer_timeout": transfer_timeout,
+            "recursive": recursive,
+        }
+    )
+    upload(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)

rucio/cli/utils.py

@@ -0,0 +1,226 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import errno
+import logging
+import os
+import signal
+import subprocess
+import sys
+import traceback
+from configparser import NoOptionError, NoSectionError
+from functools import wraps
+
+from rucio.client.client import Client
+from rucio.common.config import config_get
+from rucio.common.exception import (
+    AccessDenied,
+    CannotAuthenticate,
+    DataIdentifierAlreadyExists,
+    DataIdentifierNotFound,
+    Duplicate,
+    DuplicateContent,
+    InvalidObject,
+    InvalidRSEExpression,
+    MissingDependency,
+    RSENotFound,
+    RucioException,
+    RuleNotFound,
+    UnsupportedOperation,
+)
+from rucio.common.utils import setup_logger
+
+SUCCESS = 0
+FAILURE = 1
+
+
+def exception_handler(function):
+    verbosity = ("-v" in sys.argv) or ("--verbose" in sys.argv)
+    logger = setup_logger(module_name=__name__, logger_name="user", verbose=verbosity)
+
+    @wraps(function)
+    def new_funct(*args, **kwargs):
+        try:
+            return function(*args, **kwargs)
+        except NotImplementedError as error:
+            logger.error(f"Cannot run that operation/command combination {error}")
+            return FAILURE
+        except InvalidObject as error:
+            logger.error(error)
+            return error.error_code
+        except DataIdentifierNotFound as error:
+            logger.error(error)
+            logger.debug("This means that the Data IDentifier you provided is not known by Rucio.")
+            return error.error_code
+        except AccessDenied as error:
+            logger.error(error)
+            logger.debug("This error is a permission issue. You cannot run this command with your account.")
+            return error.error_code
+        except DataIdentifierAlreadyExists as error:
+            logger.error(error)
+            logger.debug("This means that the Data IDentifier you try to add is already registered in Rucio.")
+            return error.error_code
+        except RSENotFound as error:
+            logger.error(error)
+            logger.debug("This means that the Rucio Storage Element you provided is not known by Rucio.")
+            return error.error_code
+        except InvalidRSEExpression as error:
+            logger.error(error)
+            logger.debug("This means the RSE expression you provided is not syntactically correct.")
+            return error.error_code
+        except DuplicateContent as error:
+            logger.error(error)
+            logger.debug("This means that the DID you want to attach is already in the target DID.")
+            return error.error_code
+        except Duplicate as error:
+            logger.error(error)
+            logger.debug("This means that you are trying to add something that already exists.")
+            return error.error_code
+        except TypeError as error:
+            logger.error(error)
+            logger.debug("This means the parameter you passed has a wrong type.")
+            return FAILURE
+        except RuleNotFound as error:
+            logger.error(error)
+            logger.debug("This means the rule you specified does not exist.")
+            return error.error_code
+        except UnsupportedOperation as error:
+            logger.error(error)
+            logger.debug("This means you cannot change the status of the DID.")
+            return error.error_code
+        except MissingDependency as error:
+            logger.error(error)
+            logger.debug("This means one dependency is missing.")
+            return error.error_code
+        except KeyError as error:
+            if "x-rucio-auth-token" in str(error):
+                used_account = None
+                try:  # get the configured account from the configuration file
+                    used_account = "%s (from rucio.cfg)" % config_get("client", "account")
+                except Exception:
+                    pass
+                try:  # are we overriden by the environment?
+                    used_account = "%s (from RUCIO_ACCOUNT)" % os.environ["RUCIO_ACCOUNT"]
+                except Exception:
+                    pass
+                logger.error("Specified account %s does not have an associated identity." % used_account)
+
+            else:
+                logger.debug(traceback.format_exc())
+                contact = config_get("policy", "support", raise_exception=False)
+                support = ("Please follow up with all relevant information at: " + contact) if contact else ""
+                logger.error("\nThe object is missing this property: %s\n" 'This should never happen. Please rerun the last command with the "-v" option to gather more information.\n' "%s" % (str(error), support))
+            return FAILURE
+        except RucioException as error:
+            logger.error(error)
+            return error.error_code
+        except Exception as error:
+            if isinstance(error, IOError) and getattr(error, "errno", None) == errno.EPIPE:
+                # Ignore Broken Pipe
+                # While in python3 we can directly catch 'BrokenPipeError', in python2 it doesn't exist.
+
+                # Python flushes standard streams on exit; redirect remaining output
+                # to devnull to avoid another BrokenPipeError at shutdown
+                devnull = os.open(os.devnull, os.O_WRONLY)
+                os.dup2(devnull, sys.stdout.fileno())
+                return SUCCESS
+            logger.debug(traceback.format_exc())
+            logger.error(error)
+            contact = config_get("policy", "support", raise_exception=False)
+            support = ("If it's a problem concerning your experiment or if you're unsure what to do, please follow up at: %s\n" % contact) if contact else ""
+            contact = config_get("policy", "support_rucio", default="https://github.com/rucio/rucio/issues")
+            support += "If you're sure there is a problem with Rucio itself, please follow up at: " + contact
+            logger.error("\nRucio exited with an unexpected/unknown error.\n" 'Please rerun the last command with the "-v" option to gather more information.\n' "%s" % support)
+            return FAILURE
+
+    return new_funct
+
+
+def get_client(args, logger):
+    """
+    Returns a new client object.
+    """
+    if hasattr(args, "config") and (args.config is not None):
+        os.environ["RUCIO_CONFIG"] = args.config
+
+    if logger is None:
+        logger = setup_logger(module_name=__name__, logger_name="user", verbose=args.verbose)
+
+    if not args.auth_strategy:
+        if "RUCIO_AUTH_TYPE" in os.environ:
+            auth_type = os.environ["RUCIO_AUTH_TYPE"].lower()
+        else:
+            try:
+                auth_type = config_get("client", "auth_type").lower()
+            except (NoOptionError, NoSectionError):
+                logger.error("Cannot get AUTH_TYPE")
+                sys.exit(1)
+    else:
+        auth_type = args.auth_strategy.lower()
+
+    if auth_type in ["userpass", "saml"] and args.username is not None and args.password is not None:
+        creds = {"username": args.username, "password": args.password}
+    elif auth_type == "oidc":
+        if args.oidc_issuer:
+            args.oidc_issuer = args.oidc_issuer.lower()
+        creds = {
+            "oidc_auto": args.oidc_auto,
+            "oidc_scope": args.oidc_scope,
+            "oidc_audience": args.oidc_audience,
+            "oidc_polling": args.oidc_polling,
+            "oidc_refresh_lifetime": args.oidc_refresh_lifetime,
+            "oidc_issuer": args.oidc_issuer,
+            "oidc_username": args.oidc_username,
+            "oidc_password": args.oidc_password,
+        }
+    elif auth_type == "x509":
+        creds = {"client_cert": args.certificate, "client_key": args.client_key}
+    else:
+        creds = None
+
+    try:
+        client = Client(rucio_host=args.host, auth_host=args.auth_host, account=args.issuer, auth_type=auth_type, creds=creds, ca_cert=args.ca_certificate, timeout=args.timeout, user_agent=args.user_agent, vo=args.vo, logger=logger)
+    except CannotAuthenticate as error:
+        logger.error(error)
+        if "alert certificate expired" in str(error):
+            logger.error("The server certificate expired.")
+        elif auth_type.lower() == "x509_proxy":
+            logger.error("Please verify that your proxy is still valid and renew it if needed.")
+        sys.exit(1)
+    return client
+
+
+def signal_handler(sig, frame, logger):
+    logger.warning("You pressed Ctrl+C! Exiting gracefully")
+    child_processes = subprocess.Popen("ps -o pid --ppid %s --noheaders" % os.getpid(), shell=True, stdout=subprocess.PIPE)
+    child_processes = child_processes.stdout.read()  # type: ignore
+    for pid in child_processes.split("\n")[:-1]:  # type: ignore
+        try:
+            os.kill(int(pid), signal.SIGTERM)
+        except Exception:
+            print("Cannot kill child process")
+    sys.exit(1)
+
+
+def setup_gfal2_logger():
+    gfal2_logger = logging.getLogger("gfal2")
+    gfal2_logger.setLevel(logging.CRITICAL)
+    gfal2_logger.addHandler(logging.StreamHandler())
+
+
+class Arguments(dict):
+    """dot.notation access to dictionary attributes"""
+
+    __getattr__ = dict.get
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__

rucio/client/__init__.py

@@ -0,0 +1,15 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .client import Client  # noqa: F401