rucio 37.0.0rc1__py3-none-any.whl

rucio-37.0.0rc1.data/data/rucio/tools/reset_database.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import sys
+
+base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(base_path)
+os.chdir(base_path)
+
+from argparse import ArgumentParser  # noqa: E402
+
+from rucio.db.sqla.util import build_database, create_base_vo, create_root_account, destroy_database, drop_everything  # noqa: E402
+
+if __name__ == '__main__':
+
+    parser = ArgumentParser()
+    parser.add_argument('-d', '--drop-everything', action="store_true", default=False, help='Drop all tables and constraints')
+    args = parser.parse_args()
+
+    if args.drop_everything:
+        drop_everything()
+    else:
+        destroy_database()
+
+    build_database()
+    create_base_vo()
+    create_root_account()

rucio-37.0.0rc1.data/scripts/rucio

@@ -0,0 +1,133 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+from typing import TYPE_CHECKING, Optional
+
+from rucio.cli.bin_legacy.rucio import main as main_legacy
+from rucio.cli.command import main
+from rucio.common.utils import setup_logger
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+
+def _get_first_command(args: list[str]) -> Optional[str]:
+    return next(
+        (
+            arg for arg in args
+            if arg != 'rucio' and (arg[0] != "-" or arg in ("-h", "--help", "--version"))
+        ),
+        None
+    )
+
+
+def make_warning(logger: "Logger") -> None:
+    base_warning = "This method is being deprecated."
+    new_command = map_legacy_command()
+    if new_command is not None:
+        warning = f"{base_warning} Please replace your command with `rucio {' '.join(new_command)}`"
+    else:
+        warning = base_warning + " Please view rucio -h for an updated help menu."
+
+    logger.warning(warning)
+
+
+def map_legacy_command() -> Optional[list[str]]:
+    command = _get_first_command(sys.argv[1:])
+
+    new_command = None
+    if command not in ("-h", "--version", None):
+        command_map = {
+            "list-file-replicas": ["replica", "list", "file"],
+            "list-dataset-replicas": ["replica", "list", "dataset"],
+            "add-dataset": ["did", "add", "--type dataset"],
+            "add-container": ["did", "add", "--type container"],
+            "attach": ["did", "content", "add"],
+            "detach": ["did", "content", "remove"],
+            "ls": ["did", "list"],
+            "list-dids": ["did", "list"],
+            "list-parent-dids": ["did", "list", "--parent"],
+            "list-scopes": ["scope", "list"],
+            "close": ["did", "update", "--close"],
+            "reopen": ["did", "update", "--open"],
+            "stat": ["did", "show"],
+            "erase": ["did", "remove"],
+            "list-content": ["did", "content", "list"],
+            "list-content-history": ["did", "content", "history"],
+            "upload": ["upload"],
+            "get": ["download"],
+            "download": ["download"],
+            "get-metadata": ["did", "metadata", "list"],
+            "set-metadata": ["did", "metadata", "add"],
+            "delete-metadata": ["did", "metadata", "remove"],
+            "list-rse-usage": ["rse", "show"],
+            "list-account-usage": ["account", "limit", "list"],
+            "list-account-limits": ["account", "limit", "list"],
+            "add-rule": ["rule", "add"],
+            "delete-rule": ["rule", "remove"],
+            "rule-info": ["rule", "show"],
+            "list-rules": ["rule", "list"],
+            "list-rules-history": ["rule", "history"],
+            "update-rule": ["rule", "update"],
+            "move-rule": ["rule", "move"],
+            "list-rses": ["rse", "list"],
+            "list-suspicious-replicas": ["replica", "state", "suspicious"],
+            "list-rse-attributes": ["rse", "attribute", "list"],
+            "touch": ["did", "update", "--touch"],
+            "add-lifetime-exception": ["lifetime-exception", "add"],
+        }
+        new_command = command_map.get(command)
+
+    return new_command
+
+
+if __name__ == "__main__":
+    commands = ("account", "config", "did", "replica", "rse", "rule", "scope", "subscription", "ping", "whoami", "test-server", "lifetime-exception", "upload", "download")
+
+    parser = argparse.ArgumentParser(add_help=False)
+    # Check for legacy flag
+    parser.add_argument("--legacy", action="store_true")
+    # Check for commands in the new command list
+    parser.add_argument("-h", "--help", action="store_true")
+    parser.add_argument("--version", action="store_true")
+
+    args, _ = parser.parse_known_args()
+
+    logger = setup_logger(module_name=__name__)
+
+    if args.legacy:
+        make_warning(logger)
+        sys.argv.pop(sys.argv.index('--legacy'))
+        main_legacy()
+
+    elif (any(arg in commands for arg in sys.argv)) or args.help or args.version:
+        main()  # pylint: disable=E1120
+
+    else:
+        make_warning(logger)
+        try:
+            main_legacy()
+        # Make a custom warning - show the new help menu when invalid commands are called.
+        except argparse.ArgumentError:
+            logger.error("Invalid argument(s) - %s " % sys.argv[1:])
+            command = map_legacy_command()
+            if command is not None:
+                sys.argv = ["rucio"] + command + ["-h"]
+            else:
+                sys.argv = ["rucio", "-h"]
+
+            main()  # pylint: disable=E1120

rucio-37.0.0rc1.data/scripts/rucio-abacus-account

@@ -0,0 +1,74 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus account is a daemon to update account counters.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.abacus.account import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Abacus-Account daemon is responsible for updating account usages. It checks if there are new entries in the UpdatedAccountCounter table and updates the account counters in the AccountCounter table by adding or substrating the amount and size of files and recalculating the quotas.", epilog='''
+Upload a file::
+
+  $ rucio upload --rse MOCK --scope mock filename.txt
+
+Check account usage::
+
+  $ rucio list-account-usage username
+  +-------+---------+---------+--------------+
+  | RSE   | USAGE   | LIMIT   | QUOTA LEFT   |
+  |-------+---------+---------+--------------|
+  +-------+---------+---------+--------------+
+
+Run the daemon::
+
+  $ rucio-abacus-account --run-once
+
+Check account usage again::
+
+  $ rucio list-account-usage username
+  +-------+------------+---------+--------------+
+  | RSE   | USAGE      | LIMIT   | QUOTA LEFT   |
+  |-------+------------+---------+--------------|
+  | MOCK  | 211.724 kB | 0.000 B | 0.000 B      |
+  +-------+------------+---------+--------------+
+
+    ''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads on this process')
+    parser.add_argument("--enable-history", action="store_true", default=False, help='Record account usage into history table every hour.')
+    parser.add_argument('--sleep-time', action="store", default=10, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, threads=args.threads, fill_history_table=args.enable_history, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-abacus-collection-replica

@@ -0,0 +1,46 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus collection replica is a daemon to update collection replicas.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.abacus.collection_replica import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="Abacus collection replica is a daemon to update the state and counters of collection replicas.")
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads on this process')
+    parser.add_argument('--sleep-time', action="store", default=10, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    parser.add_argument("--limit", action="store", default=1000, type=int, help='Size of the chunk')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, threads=args.threads, sleep_time=args.sleep_time, limit=args.limit)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-abacus-rse

@@ -0,0 +1,78 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Abacus rse is a daemon to update rse counters.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.abacus.rse import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Abacus-RSE daemon is responsible for updating RSE usages. It checks if there are new entries in the UpdatedRSECounter table and updates the RSE counter in the RSECounter table by adding or substrating the amount of files and the size.", epilog='''
+Upload a file to your RSE::
+
+  $ rucio upload --rse MOCK --scope mock filename.txt
+
+Check RSE usage::
+
+  $ rucio list-rse-usage MOCK
+  USAGE:
+  ------
+     files: 0
+     used: 0.000 B
+     rse: MOCK
+     updated_at: 2018-11-30 14:28:34
+     source: rucio
+  ------
+
+Run the daemon::
+
+  $ rucio-abacus-rse --run-once
+
+Check RSE usage again::
+
+  $ rucio list-rse-usage MOCK
+  USAGE:
+  ------
+      files: 1
+      used: 213.481 kB
+      rse: MOCK
+      updated_at: 2018-12-03 08:58:33
+      source: rucio
+  ------
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads on this process')
+    parser.add_argument("--enable-history", action="store_true", default=False, help='Record RSE usage into history table every hour.')
+    parser.add_argument('--sleep-time', action="store", default=10, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, threads=args.threads, fill_history_table=args.enable_history, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-admin

@@ -0,0 +1,97 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from rucio.cli.bin_legacy.rucio_admin import main as main_legacy
+from rucio.common.utils import setup_logger
+
+
+def make_warning():
+    logger = setup_logger(module_name=__name__)
+
+    base_warning = "This method is being deprecated."
+    args = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--help", "--version")]
+    try:
+        first_command = args[1]
+    except IndexError:
+        first_command = None
+    try:
+        second_command = args[2]
+    except IndexError:
+        second_command = "-h"
+
+    if (first_command not in ("-h", "--version", "--help")) and (first_command is not None):
+        command_map = {
+            "data": {"import": "upload", "export": "download"},
+            "account": {
+                "-h": "account -h",
+                "add": "account add",
+                "delete": "account remove",
+                "info": "account show",
+                "update": "account update",
+                "set-limits": "account limit add",
+                "get-limits": "account limit list",
+                "delete-limits": "account limit remove",
+                "ban": "account update --ban",
+                "unban": "account update --unban",
+                "list-attributes": "account attribute list",
+                "add-attribute": "account attribute add",
+                "delete-attribute": "account attribute remove",
+            },
+            "identity": {"-h": "account identity -h", "add": "account identity add", "delete": "account identity remove"},
+            "rse": {
+                "-h": "rse -h",
+                "add": "rse add",
+                "list": "rse list",
+                "update": "rse update",
+                "info": "rse show",
+                "set-attribute": "rse attribute add",
+                "delete-attribute": "rse attribute remove",
+                "delete-distance": "rse distance remove",
+                "get-distance": "rse distance show",
+                "set-distance": "rse distance update",
+                "get-attribute": "rse attribute list",
+                "add-protocol": "rse protocol add",
+                "delete-protocol": "rse protocol remove",
+                "delete": "rse remove",
+                "add-qos-policy": "rse qos add",
+                "add-distance": "rse distance add",
+                "delete-qos-policy": "rse qos remove",
+                "list-qos-policies": "rse qos list",
+                "set-limit": "rse limit add",
+                "delete-limit": "rse limit remove",
+            },
+            "scope": {"-h": "scope -h", "add": "scope add", "list": "scope list"},
+            "config": {"-h": "config -h", "get": "config list", "set": "config add", "delete": "config remove"},
+            "subscription": {"-h": "subscription -h", "add": "subscription add", "list": "subscription", "update": "subscription update", "reevaluate": "subscription touch"},
+            "replicas": {"-h": "replica -h", "quarantine": "replica state update quarantine", "declare-bad": "replica state update bad", "declare-temporary-unavailable": "replica state update unavailable", "set-tombstone": "replica remove"},
+        }
+        try:
+            new_command = command_map[first_command]
+            new_command = new_command[second_command]
+        except KeyError:
+            new_command = "-h"
+
+        warning = f"{base_warning} Please replace your command with `rucio {new_command}`"
+    else:
+        warning = base_warning + " Please view rucio -h for an updated help menu."
+
+    logger.warning(warning)
+
+
+if __name__ == "__main__":
+    make_warning()
+    main_legacy()

rucio-37.0.0rc1.data/scripts/rucio-atropos

@@ -0,0 +1,60 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Atropos Daemon : End the life of the rules according to the Lifetime Model
+"""
+
+import argparse
+import datetime
+import signal
+
+from rucio.daemons.atropos.atropos import run, stop
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description='The Atropos Daemon is responsible for the deletion of rules with expired eol_at (end the life at), according to the Lifetime Model. Once the rule is deleted, the replicas covered by the rule will not be protected anymore and eventually, will be deleted by another daemon.')  # noqa E501
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--dry-run", action="store_true", default=False, help='Dry run mode')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--grace-period", action="store", default=86400, type=int, help='Grace period for the rules. In seconds !!!')
+    parser.add_argument("--date-check", action="store", help='Date when the lifetime model will be applied. Cannot be used for a date in the future if dry-run is not enabled',
+                        default=datetime.datetime.utcnow(), type=lambda d: datetime.datetime.strptime(d, '%Y-%m-%d'))
+    parser.add_argument("--unlock-rules", action="store_true", default=False, help='Automatically unlock affected rules')
+    parser.add_argument("--spread-period", action="store", default=0, type=int, help='Set the rules to randomly expire over a period (in seconds). Uses a uniform distribution')
+    parser.add_argument("--purge-replicas", action="store_true", default=False, help='Set the replicas to be deleted instead of secondarised')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads, date_check=args.date_check,
+            dry_run=args.dry_run, grace_period=args.grace_period,
+            once=args.run_once, unlock=args.unlock_rules,
+            spread_period=args.spread_period,
+            purge_replicas=args.purge_replicas,
+            sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()