rucio 32.8.6__py3-none-any.whl

rucio-32.8.6.data/scripts/rucio-light-reaper

@@ -0,0 +1,53 @@
+#!python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Reaper is a daemon to manage sub-file deletion
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.reaper.light_reaper import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The light-reaper is responsible of deletion of temporary files")
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--total-workers", action="store", default=1, type=int, help='Total number of workers per process')
+    parser.add_argument("--chunk-size", action="store", default=10, type=int, help='Chunk size')
+    parser.add_argument("--scheme", action="store", default=None, type=str, help='Force the reaper to use a particular protocol, e.g., mock.')
+    parser.add_argument('--rses', nargs='+', type=str, help='List of RSEs')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str, help='RSEs expression to exclude RSEs')
+    parser.add_argument('--include-rses', action="store", default=None, type=str, help='RSEs expression to include RSEs')
+    parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == '__main__':
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(total_workers=args.total_workers, chunk_size=args.chunk_size,
+            once=args.run_once, scheme=args.scheme, rses=args.rses,
+            exclude_rses=args.exclude_rses, include_rses=args.include_rses, vos=args.vos, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-32.8.6.data/scripts/rucio-minos

@@ -0,0 +1,54 @@
+#!python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Minos Daemon :
+Minos is the name of one of the judge on the underworld in the greek mythology.
+The role of the daemon is get as input the list of PFNs declared bad
+and to class them into 2 categories :
+- The ones temporary available
+- The ones that have a real problem and that will need to be recovered by the necromancer
+'''
+
+import argparse
+import signal
+
+from rucio.daemons.badreplicas.minos import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The role of the daemon is get as input the list of PFNs declared bad and to classify them into 2 categories: the temporary available ones, and the ones that have a real problem and that will need to be recovered by the necromancer.")  # noqa E501
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Seconds to sleep if few requests')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads, bulk=args.bulk, once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-32.8.6.data/scripts/rucio-minos-temporary-expiration

@@ -0,0 +1,51 @@
+#!python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Minos Daemon Temporary Unavailable:
+This special type of minos daemon lists the expired TEMPORARY_UNAVAILABLE replicas and
+puts them back into AVAILABLE state.
+'''
+
+import argparse
+import signal
+
+from rucio.daemons.badreplicas.minos_temporary_expiration import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="This special type of minos daemon lists the expired TEMPORARY_UNAVAILABLE replicas and puts them back into AVAILABLE state.")
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Seconds to sleep if few requests')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads, bulk=args.bulk, once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-32.8.6.data/scripts/rucio-necromancer

@@ -0,0 +1,121 @@
+#!python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Necromancer Daemon : Bring the dead files back to life
+'''
+
+import argparse
+import signal
+
+from rucio.daemons.badreplicas.necromancer import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Necromancer daemon is responsible for managing bad replicas. If a replica that got declared bad has other replicas, it will try to recover it by requesting a new transfer. If there are no replicas anymore, then the file gets marked as lost.", epilog='''
+Lost replica:
+In this example the file gets uploaded and will only have this replica as there are no replication rules. If it gets declared bad, there will be no replica to recover from.
+Therefor the replica gets marked as lost.
+
+Upload a file::
+
+  $ rucio upload --scope mock --rse MOCK --name file filename.txt
+
+Check replicas::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+  | SCOPE   | NAME   | FILESIZE   |   ADLER32 | RSE: REPLICA                                            |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK: file://localhost:0/tmp/rucio_rse/mock/fb/d1/file  |
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+
+Declare it as bad::
+
+  $ rucio-admin replicas declare-bad file://localhost:0/tmp/rucio_rse/mock/fb/d1/file --reason 'bad'
+
+Run the daemon::
+
+  $ rucio-necromancer --run-once
+
+Check replicas again::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+----------------+
+  | SCOPE   | NAME   | FILESIZE   | ADLER32   | RSE: REPLICA   |
+  |---------+--------+------------+-----------+----------------|
+  +---------+--------+------------+-----------+----------------+
+
+Bad replica:
+In this example the file gets uploaded and will have two replicas. If it gets declared bad, then the daemon will try to recover it from the second replica.
+
+Upload a file and replicate it::
+
+  $ rucio upload --scope mock --rse MOCK filename.txt
+  $ rucio add-rule mock:file 1 MOCK2
+  $ rucio-conveyor-submitter --run-once
+
+Check replicas::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+  | SCOPE   | NAME   | FILESIZE   |   ADLER32 | RSE: REPLICA                                            |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK: file://localhost:0/tmp/rucio_rse/mock/fb/d1/file  |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK2: file://localhost:1/tmp/rucio_rse/mock/fb/d1/file |
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+
+Declare one replica as bad::
+
+  $ rucio-admin replicas declare-bad file://localhost:1/tmp/rucio_rse/mock/fb/d1/file --reason 'bad'
+
+Run the daemon::
+
+  $ rucio-necromancer --run-once
+
+Check replicas again::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+  | SCOPE   | NAME   | FILESIZE   |   ADLER32 | RSE: REPLICA                                            |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK: file://localhost:0/tmp/rucio_rse/mock/fb/d1/file  |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK2: file://localhost:1/tmp/rucio_rse/mock/fb/d1/file |
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads, bulk=args.bulk, once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-32.8.6.data/scripts/rucio-oauth-manager

@@ -0,0 +1,64 @@
+#!python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+OAuth Manager is a daemon which is reponsible for:
+- deletion of expired access tokens (in case there is a valid refresh token, expired access tokens will be kept until refresh_token expires as well.)
+- deletion of expired OAuth session parameters
+- refreshing access tokens via their refresh tokens.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.oauthmanager.oauthmanager import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description='''
+
+OAuth Manager is a daemon which is reponsible for:
+- deletion of expired access tokens (in case there is a valid refresh token,
+  expired access tokens will be kept until refresh_token expires as well.)
+- deletion of expired OAuth session parameters
+- refreshing access tokens via their refresh tokens.
+
+These 3 actions run consequently one after another in a loop with a sleeptime of 'looprate' seconds.
+The maximum number of DB rows (tokens, parameters, refresh tokens) on which the script will operate
+can be specified by 'maxrows' parameter.
+
+''')  # NOQA: E501
+    parser.add_argument("--max-rows", action="store", default=1000, help='Max number of DB rows to deal with per operation.')
+    parser.add_argument("--loop-rate", action="store", default=300, help='OBSOLETE, USE --sleep-time INSTEAD. The number of seconds the daemon will wait before running next loop of operations.')
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads for this process')
+    parser.add_argument("--sleep-time", action="store", default=300, type=int, help='The number of seconds the daemon will wait before running next loop of operations.')
+
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    PARSER = get_parser()
+    ARGS = PARSER.parse_args()
+    try:
+        run(once=ARGS.run_once, max_rows=ARGS.max_rows, threads=ARGS.threads,
+            sleep_time=ARGS.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-32.8.6.data/scripts/rucio-reaper

@@ -0,0 +1,84 @@
+#!python
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+'''
+
+import argparse
+import signal
+
+from rucio.common.utils import StoreAndDeprecateWarningAction
+from rucio.daemons.reaper.reaper import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int,
+                        help='Concurrency control: number of threads')
+    parser.add_argument("--chunk_size", "--chunk-size", new_option_string="--chunk-size", action=StoreAndDeprecateWarningAction, default=100, type=int,
+                        help='The size used for a bulk deletion on on RSE')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int,
+                        help='Minimum time between 2 consecutive cycles')
+    parser.add_argument('--greedy', action="store_true", default=False,
+                        help='To enable greedy mode. Deprecated, kept for compatibility reasons. Recommended way is to use RSE attribute greedyDeletion=True')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str,
+                        help='RSE expression to exclude')
+    parser.add_argument('--include-rses', action="store", default=None, type=str,
+                        help='RSE expression to include')
+    parser.add_argument('--rses', nargs='+', type=str,
+                        help='Explicit list of RSEs to include. If empty, it considers all RSEs')
+    parser.add_argument('--vos', nargs='+', type=str,
+                        help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument("--delay-seconds", action="store", default=600, type=int,
+                        help='The delay (seconds) to query replicas in BEING_DELETED state.')
+    parser.add_argument('--scheme', action="store", default=None, type=str,
+                        help='Force the reaper to use a particular protocol/scheme, e.g., mock')
+    parser.add_argument("--auto_exclude_threshold", "--auto-exclude-threshhold", new_option_string="--auto-exclude-threshhold", action=StoreAndDeprecateWarningAction, default=100, type=int,
+                        help='Number of service unavailable exceptions after which the RSE gets temporarily excluded.')
+    parser.add_argument("--auto_exclude_timeout", "--auto-exclude-timeout", new_option_string="--auto-exclude-timeout", action=StoreAndDeprecateWarningAction, default=600, type=int,
+                        help='Timeout for temporarily excluded RSEs.')
+
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads,
+            chunk_size=args.chunk_size,
+            include_rses=args.include_rses,
+            exclude_rses=args.exclude_rses,
+            rses=args.rses,
+            vos=args.vos,
+            once=args.run_once,
+            greedy=args.greedy,
+            scheme=args.scheme,
+            delay_seconds=args.delay_seconds,
+            sleep_time=args.sleep_time,
+            auto_exclude_threshold=args.auto_exclude_threshold,
+            auto_exclude_timeout=args.auto_exclude_timeout)
+    except KeyboardInterrupt:
+        stop()