rucio 35.7.0__py3-none-any.whl

rucio-35.7.0.data/scripts/rucio-minos-temporary-expiration

@@ -0,0 +1,50 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Minos Daemon Temporary Unavailable:
+This special type of minos daemon lists the expired TEMPORARY_UNAVAILABLE replicas and
+puts them back into AVAILABLE state.
+'''
+
+import argparse
+import signal
+
+from rucio.daemons.badreplicas.minos_temporary_expiration import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="This special type of minos daemon lists the expired TEMPORARY_UNAVAILABLE replicas and puts them back into AVAILABLE state.")
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Seconds to sleep if few requests')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads, bulk=args.bulk, once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-necromancer

@@ -0,0 +1,120 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Necromancer Daemon : Bring the dead files back to life
+'''
+
+import argparse
+import signal
+
+from rucio.daemons.badreplicas.necromancer import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Necromancer daemon is responsible for managing bad replicas. If a replica that got declared bad has other replicas, it will try to recover it by requesting a new transfer. If there are no replicas anymore, then the file gets marked as lost.", epilog='''
+Lost replica:
+In this example the file gets uploaded and will only have this replica as there are no replication rules. If it gets declared bad, there will be no replica to recover from.
+Therefore the replica gets marked as lost.
+
+Upload a file::
+
+  $ rucio upload --scope mock --rse MOCK --name file filename.txt
+
+Check replicas::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+  | SCOPE   | NAME   | FILESIZE   |   ADLER32 | RSE: REPLICA                                            |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK: file://localhost:0/tmp/rucio_rse/mock/fb/d1/file  |
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+
+Declare it as bad::
+
+  $ rucio-admin replicas declare-bad file://localhost:0/tmp/rucio_rse/mock/fb/d1/file --reason 'bad'
+
+Run the daemon::
+
+  $ rucio-necromancer --run-once
+
+Check replicas again::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+----------------+
+  | SCOPE   | NAME   | FILESIZE   | ADLER32   | RSE: REPLICA   |
+  |---------+--------+------------+-----------+----------------|
+  +---------+--------+------------+-----------+----------------+
+
+Bad replica:
+In this example the file gets uploaded and will have two replicas. If it gets declared bad, then the daemon will try to recover it from the second replica.
+
+Upload a file and replicate it::
+
+  $ rucio upload --scope mock --rse MOCK filename.txt
+  $ rucio add-rule mock:file 1 MOCK2
+  $ rucio-conveyor-submitter --run-once
+
+Check replicas::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+  | SCOPE   | NAME   | FILESIZE   |   ADLER32 | RSE: REPLICA                                            |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK: file://localhost:0/tmp/rucio_rse/mock/fb/d1/file  |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK2: file://localhost:1/tmp/rucio_rse/mock/fb/d1/file |
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+
+Declare one replica as bad::
+
+  $ rucio-admin replicas declare-bad file://localhost:1/tmp/rucio_rse/mock/fb/d1/file --reason 'bad'
+
+Run the daemon::
+
+  $ rucio-necromancer --run-once
+
+Check replicas again::
+
+  $ rucio list-file-replicas mock:file
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+  | SCOPE   | NAME   | FILESIZE   |   ADLER32 | RSE: REPLICA                                            |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK: file://localhost:0/tmp/rucio_rse/mock/fb/d1/file  |
+  |---------+--------+------------+-----------+---------------------------------------------------------|
+  | mock    | file   | 149.000 B  |    948240 | MOCK2: file://localhost:1/tmp/rucio_rse/mock/fb/d1/file |
+  +---------+--------+------------+-----------+---------------------------------------------------------+
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads, bulk=args.bulk, once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-oauth-manager

@@ -0,0 +1,63 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+OAuth Manager is a daemon which is responsible for:
+- deletion of expired access tokens (in case there is a valid refresh token, expired access tokens will be kept until refresh_token expires as well.)
+- deletion of expired OAuth session parameters
+- refreshing access tokens via their refresh tokens.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.oauthmanager.oauthmanager import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description='''
+
+OAuth Manager is a daemon which is responsible for:
+- deletion of expired access tokens (in case there is a valid refresh token,
+  expired access tokens will be kept until refresh_token expires as well.)
+- deletion of expired OAuth session parameters
+- refreshing access tokens via their refresh tokens.
+
+These 3 actions run consequently one after another in a loop with a sleeptime of 'looprate' seconds.
+The maximum number of DB rows (tokens, parameters, refresh tokens) on which the script will operate
+can be specified by 'maxrows' parameter.
+
+''')  # NOQA: E501
+    parser.add_argument("--max-rows", action="store", default=1000, help='Max number of DB rows to deal with per operation.')
+    parser.add_argument("--loop-rate", action="store", default=300, help='OBSOLETE, USE --sleep-time INSTEAD. The number of seconds the daemon will wait before running next loop of operations.')
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads for this process')
+    parser.add_argument("--sleep-time", action="store", default=300, type=int, help='The number of seconds the daemon will wait before running next loop of operations.')
+
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    PARSER = get_parser()
+    ARGS = PARSER.parse_args()
+    try:
+        run(once=ARGS.run_once, max_rows=ARGS.max_rows, threads=ARGS.threads,
+            sleep_time=ARGS.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-reaper

@@ -0,0 +1,83 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+'''
+
+import argparse
+import signal
+
+from rucio.common.utils import StoreAndDeprecateWarningAction
+from rucio.daemons.reaper.reaper import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int,
+                        help='Concurrency control: number of threads')
+    parser.add_argument("--chunk_size", "--chunk-size", new_option_string="--chunk-size", action=StoreAndDeprecateWarningAction, default=100, type=int,
+                        help='The size used for a bulk deletion on on RSE')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int,
+                        help='Minimum time between 2 consecutive cycles')
+    parser.add_argument('--greedy', action="store_true", default=False,
+                        help='To enable greedy mode. Deprecated, kept for compatibility reasons. Recommended way is to use RSE attribute greedyDeletion=True')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str,
+                        help='RSE expression to exclude')
+    parser.add_argument('--include-rses', action="store", default=None, type=str,
+                        help='RSE expression to include')
+    parser.add_argument('--rses', nargs='+', type=str,
+                        help='Explicit list of RSEs to include. If empty, it considers all RSEs')
+    parser.add_argument('--vos', nargs='+', type=str,
+                        help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument("--delay-seconds", action="store", default=600, type=int,
+                        help='The delay (seconds) to query replicas in BEING_DELETED state.')
+    parser.add_argument('--scheme', action="store", default=None, type=str,
+                        help='Force the reaper to use a particular protocol/scheme, e.g., mock')
+    parser.add_argument("--auto_exclude_threshold", "--auto-exclude-threshhold", new_option_string="--auto-exclude-threshhold", action=StoreAndDeprecateWarningAction, default=100, type=int,
+                        help='Number of service unavailable exceptions after which the RSE gets temporarily excluded.')
+    parser.add_argument("--auto_exclude_timeout", "--auto-exclude-timeout", new_option_string="--auto-exclude-timeout", action=StoreAndDeprecateWarningAction, default=600, type=int,
+                        help='Timeout for temporarily excluded RSEs.')
+
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads,
+            chunk_size=args.chunk_size,
+            include_rses=args.include_rses,
+            exclude_rses=args.exclude_rses,
+            rses=args.rses,
+            vos=args.vos,
+            once=args.run_once,
+            greedy=args.greedy,
+            scheme=args.scheme,
+            delay_seconds=args.delay_seconds,
+            sleep_time=args.sleep_time,
+            auto_exclude_threshold=args.auto_exclude_threshold,
+            auto_exclude_timeout=args.auto_exclude_timeout)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-replica-recoverer

@@ -0,0 +1,248 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Replica-Recoverer is a daemon that declares suspicious replicas as bad if they are found available on other RSE.
+Consequently, automatic replica recovery is triggered via necromancer daemon, which creates a rule for such bad replicas.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.replicarecoverer.suspicious_replica_recoverer import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description='''
+Replica-Recoverer is a daemon that declares suspicious replicas
+that are available on other RSE as bad. Consequently, automatic
+replica recovery is triggered via necromancer daemon,
+which creates a rule for such bad replicas.''', epilog='''
+
+Preparing RSEs and DIDs for testing this daemon from rucio docker:
+-------------------------------------------------------------------
+
+$ sudo docker exec -it dev_rucio_1 /bin/bash
+
+Adding the RSEs to rucio DB:
+
+$ rucio-admin rse add MOCK_SUSPICIOUS
+$ rucio-admin rse set-attribute --rse MOCK_SUSPICIOUS --key backend_type --value file
+$ rucio-admin rse set-attribute --rse MOCK_SUSPICIOUS --key storage_usage_tool --value 'rucio.rse.protocols.posix.Default.getSpace'
+$ rucio-admin rse add-protocol --hostname localhost --scheme file --prefix '/tmp/rucio_rse1/' --space-token 'ATLASDATADISK1' --web-service-path '/srm/managerv2?SFN=' --domain-json '{ "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1}}' --impl 'rucio.rse.protocols.posix.Default'  MOCK_SUSPICIOUS
+$ rucio-admin rse info MOCK_SUSPICIOUS
+
+$ rucio-admin rse add MOCK_RECOVERY
+$ rucio-admin rse set-attribute --rse MOCK_RECOVERY --key backend_type --value POSIX
+$ rucio-admin rse set-attribute --rse MOCK_RECOVERY --key storage_usage_tool --value 'rucio.rse.protocols.posix.Default.getSpace'
+$ rucio-admin rse add-protocol --hostname localhost --scheme file --prefix '/tmp/rucio_rse2/' --space-token 'ATLASDATADISK2' --web-service-path '/srm/managerv2?SFN=' --domain-json '{ "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1}}' --impl 'rucio.rse.protocols.posix.Default'  MOCK_RECOVERY
+$ rucio-admin rse info MOCK_RECOVERY
+
+For testing, we create the following files:
+-------------------------------------------------------------------
+
+Name                                  MOCK_RECOVERY       MOCK_SUSPICIOUS    BAD (on MOCK_SUSPICIOUS)    SUSPICIOUS (on MOCK_SUSPICIOUS)
+file_available_suspicious             yes                 yes                no                          yes
+file_available_suspicious_and_bad     yes                 yes                yes                         yes
+file_notavailable_suspicious          unavailable         yes                no                          yes
+
+Only file_available_suspicious should be the one on which the daemon takes action and declares it as bad.
+
+$ id0=`uuidgen`
+$ id1=`uuidgen`
+$ id2=`uuidgen`
+$ id3=`uuidgen`
+$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS  (11 times)" > /tmp/file_available_suspicious'_'$id1
+$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS  (11 times) and 1 time bad/deleted/lost on MOCK_SUSPICIOUS" > /tmp/file_available_suspicious_and_bad'_'$id2
+$ echo "file declared as unavailable on MOCK_RECOVERY and declared as suspicious 11 times on MOCK_SUSPICIOUS" > /tmp/file_notavailable_suspicious'_'$id3
+
+Uploading the files created above to rucio:
+-------------------------------------------------------------------
+
+rucio add-dataset mock:dataset_of_suspicious_replicas'_'$id0
+
+Added mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778
+
+$ rucio add-rule mock:dataset_of_suspicious_replicas'_'$id0 1 MOCK_SUSPICIOUS
+$ rucio add-rule --source-replica-expression MOCK_SUSPICIOUS mock:dataset_of_suspicious_replicas'_'$id0 1 MOCK_RECOVERY
+$ rucio list-rules mock:dataset_of_suspicious_replicas'_'$id0
+
+ID                                ACCOUNT    SCOPE:NAME                                                                STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
+--------------------------------  ---------  ------------------------------------------------------------------------  ----------------------  ----------------  --------  ---------------  -------------------
+2a1a078b66ca4e209cc20a5826125334  root       mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778  OK[0/0/0]               MOCK_RECOVERY            1                   2019-02-19 14:12:30
+8c15d2f8e94a459a86a488055a10d068  root       mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778  OK[0/0/0]               MOCK_SUSPICIOUS          1                   2019-02-19 14:12:28
+
+$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_available_suspicious'_'$id1 /tmp/file_available_suspicious'_'$id1
+$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_available_suspicious_and_bad'_'$id2 /tmp/file_available_suspicious_and_bad'_'$id2
+$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_notavailable_suspicious'_'$id3 /tmp/file_notavailable_suspicious'_'$id3
+$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_available_suspicious'_'$id1 /tmp/file_available_suspicious'_'$id1
+$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_available_suspicious_and_bad'_'$id2 /tmp/file_available_suspicious_and_bad'_'$id2
+$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_notavailable_suspicious'_'$id3 /tmp/file_notavailable_suspicious'_'$id3
+[...]
+$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_available_suspicious'_'$id1
+$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_available_suspicious_and_bad'_'$id2
+$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_notavailable_suspicious'_'$id3
+[...]
+
+$ rucio list-file-replicas mock:file_available_suspicious'_'$id1
++---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------+
+| SCOPE   | NAME                                                           | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                 |
+|---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------|
+| mock    | file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e | 87.000 B   | 206b1c91  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e |
+| mock    | file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e | 87.000 B   | 206b1c91  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e   |
++---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------+
+
+$ rucio list-file-replicas mock:file_available_suspicious_and_bad'_'$id2
++---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------+
+| SCOPE   | NAME                                                                   | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                         |
+|---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------|
+| mock    | file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 | 134.000 B  | dfdf2bff  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 |
+| mock    | file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 | 134.000 B  | dfdf2bff  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835   |
++---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------+
+
+$ rucio list-file-replicas mock:file_notavailable_suspicious'_'$id3
++---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------+
+| SCOPE   | NAME                                                              | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                    |
+|---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------|
+| mock    | file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f | 101.000 B  | 0c14223f  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f |
+| mock    | file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f | 101.000 B  | 0c14223f  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f   |
++---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------+
+
+Modifying the file statuses in the DB:
+--------------------------------------
+
+$ python
+
+# the paths below point to MOCK_SUSPICIOUS RSE (.../rucio_rse1)
+
+$$ file1 = ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e',]
+$$ file2 = ['file://localhost:0/tmp/rucio_rse1/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835',]
+$$ file3 = ['file://localhost:0/tmp/rucio_rse1/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f' ]
+
+$$ from rucio.client.replicaclient import ReplicaClient
+$$ replica_client = ReplicaClient()
+$$ import time
+$$ for i in range(11):
+       replica_client.declare_suspicious_file_replicas(file1, 'This is a good reason')
+       replica_client.declare_suspicious_file_replicas(file2, 'This is a good reason')
+       replica_client.declare_suspicious_file_replicas(file3, 'This is a good reason')
+       time.sleep(1)
+
+#  Declaring file2 bad on MOCK_SUSPICIOUS:
+   ---------------------------------------
+
+$$ replica_client.declare_bad_file_replicas(file2, 'This is a good reason')
+
+#  Update replica state of 'file_notavailable_suspicious'_'$id1' on MOCK_RECOVERY to 'UNAVAILABLE'
+#  (change the file name below according to the info from rucio !):
+   -----------------------------------------------------------------------------------------------
+
+$$ replica_client.update_replicas_states('MOCK_RECOVERY', [{'scope':'mock', 'name':'file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f', 'state':'U'}])
+
+
+# Checking the results of the file status changes:
+  ------------------------------------------------
+$$ from rucio.core.replica import get_suspicious_files
+$$ from datetime import datetime, timedelta
+$$ from_date = datetime.now() - timedelta(days=3)
+$$ from rucio.core.replica import list_bad_replicas_status
+
+$$ get_suspicious_files('MOCK_SUSPICIOUS',from_date,10)
+
+[{'created_at': datetime.datetime(2019, 2, 19, 14, 12, 56), 'scope': 'mock', 'cnt': 11L, 'name': 'file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f', 'rse': 'MOCK_SUSPICIOUS'},
+ {'created_at': datetime.datetime(2019, 2, 19, 14, 12, 48), 'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
+
+$$ list_bad_replicas_status(rse='MOCK_SUSPICIOUS', younger_than=from_date)
+
+[{'name': 'file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835', 'rse': 'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'updated_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'state': BAD, 'scope': 'mock'}]
+
+$$ exit()
+
+Run the daemon:
+---------------
+
+  $ python bin/rucio-replica-recoverer --run-once --rse-expression='MOCK_SUSPICIOUS'
+
+Terminal output:
+----------------
+
+2019-02-19 14:39:24,114 709 INFO  replica_recoverer[0/0]: ready to query replicas at RSEs like *MOCK*, declared as suspicious in the last 3 days at least 10 times and which are available on other RSEs.
+2019-02-19 14:39:24,124 709 INFO  replica_recoverer[0/0]: suspicious replica query took 0.0101511478424 seconds, total of 1 replicas were found. [{'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
+2019-02-19 14:39:24,125 709 INFO  replica_recoverer[0/0]: looking for replica pfns.
+2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: found 1/1 pfns (took 0.035572052002 seconds) - declaring them as bad replicas now.
+2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: ready to declare 1 bad replica(s) on MOCK_SUSPICIOUS: ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e'].
+2019-02-19 14:39:24,188 709 INFO  replica_recoverer[0/0]: finished declaring bad replicas on MOCK_SUSPICIOUS.
+2019-02-19 14:39:24,192 709 INFO  replica_recoverer[0/0]: graceful stop done
+
+
+# Checking the results of the file status changes:
+  ------------------------------------------------
+$ python
+
+$$ from rucio.core.replica import get_suspicious_files
+$$ from datetime import datetime, timedelta
+$$ from_date = datetime.now() - timedelta(days=3)
+$$ from rucio.core.replica import list_bad_replicas_status
+$$ from rucio.common.constants import SuspiciousAvailability
+
+$$ get_suspicious_files('MOCK_SUSPICIOUS',younger_than=from_date, nattempts=10, is_suspicious=True, available_elsewhere=SuspiciousAvailability["EXIST_COPIES"].value)
+
+>>> get_suspicious_files('MOCK_SUSPICIOUS',younger_than=from_date, nattempts=10, is_suspicious=True, available_elsewhere=SuspiciousAvailability["EXIST_COPIES"].value)
+[]
+
+$$ list_bad_replicas_status(rse='MOCK_SUSPICIOUS', younger_than=from_date)
+
+[{'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 39, 24), 'updated_at': datetime.datetime(2019, 2, 19, 14, 39, 24), 'state': BAD, 'scope': 'mock'},
+ {'name': 'file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835', 'rse': 'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'updated_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'state': BAD, 'scope': 'mock'}]
+
+$$ exit()
+
+When run in multi-VO mode, by default the daemon will run on RSEs from all VOs::
+
+  $ rucio-replica-recoverer --run-once
+  2020-07-28 15:15:14,151 5461    INFO    replica_recoverer: This instance will work on VOs: def, abc, xyz, 123
+
+By using the ``--vos`` argument only the VO or VOs specified will be affected::
+
+  $ rucio-replica-recoverer --run-once --vos abc xyz
+  2020-07-28 15:16:36,066 5474    INFO    replica_recoverer: This instance will work on VOs: abc, xyz
+
+Note that attempting the use the ``--vos`` argument when in single-VO mode will have no affect::
+
+  $ rucio-replica-recoverer --run-once --vos abc xyz
+  2020-07-28 15:21:33,349 5488    WARNING Ignoring argument vos, this is only applicable in a multi-VO setup.
+''', formatter_class=argparse.RawDescriptionHelpFormatter)  # NOQA: E501
+    parser.add_argument("--nattempts", action="store", default=5, type=int, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 5.')
+    parser.add_argument("--younger-than", action="store", default=5, type=int, help='Consider all file replicas logged in bad_replicas table since specified number of younger-than days. Default value is 5.')
+    parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
+    parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, type=int, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
+    parser.add_argument('--json-file-name', action="store", default="/opt/rucio/etc/suspicious_replica_recoverer.json", type=str, help='Name of the json file that that contains the policies which will be used by the suspicious replica recoverer.')
+    parser.add_argument('--sleep-time', action="store", default=3600, type=int, help='Concurrency control: Thread sleep time after each chunk of work.')
+    parser.add_argument('--active-mode', action="store_true", default=False, help='If NOT specified, the daemon will run without taking any actions on any files. In either case, the log file will be produced normally.')
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    PARSER = get_parser()
+    ARGS = PARSER.parse_args()
+    try:
+        run(once=ARGS.run_once, younger_than=ARGS.younger_than, nattempts=ARGS.nattempts, vos=ARGS.vos, limit_suspicious_files_on_rse=ARGS.limit_suspicious_files_on_rse, json_file_name=ARGS.json_file_name, sleep_time=ARGS.sleep_time, active_mode=ARGS.active_mode)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-rse-decommissioner

@@ -0,0 +1,66 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+RSE-Decommissioner is a daemon that clears RSEs scheduled for decommissioning.
+The actual operations (e.g. just delete all replicas, move the replicas to
+specific RSEs, etc.) that are performed on the RSEs depend on the
+"decommissioning profile", which must be specified for each RSE through RSE
+attributes.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.rsedecommissioner.rse_decommissioner import run, stop
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """Construct and return the argparse parser."""
+
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        epilog="""\
+To mark an RSE for decommissioning
+----------------------------------
+
+Clearing with the generic "delete all" profile:
+
+$ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
+              --value profile=generic_delete
+
+Clearing with the generic "move all" profile:
+
+$ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
+              --value profile=generic_move,destination=ANOTHER_RSE""",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument('--run-once', dest='run_once', action='store_true',
+                        help='One iteration only.')
+    parser.add_argument('--sleep-time', dest='sleep_time', action='store',
+                        default=86400, type=int,
+                        help='Concurrency control: thread sleep time after each'
+                        ' chunk of work')
+    return parser
+
+
+if __name__ == '__main__':
+    signal.signal(signal.SIGTERM, stop)
+    args = get_parser().parse_args()
+    try:
+        run(once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()