rucio 37.0.0rc1__py3-none-any.whl

rucio-37.0.0rc1.data/scripts/rucio-reaper

@@ -0,0 +1,83 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Reaper is a daemon to manage file deletion.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.reaper.reaper import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int,
+                        help='Concurrency control: number of threads')
+    parser.add_argument("--chunk-size", action="store", default=100, type=int,
+                        help='The size used for a bulk deletion on on RSE')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int,
+                        help='Minimum time between 2 consecutive cycles')
+    parser.add_argument('--greedy', action="store_true", default=False,
+                        help='To enable greedy mode. Deprecated, kept for compatibility reasons. Recommended way is to use RSE attribute greedyDeletion=True')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str,
+                        help='RSE expression to exclude')
+    parser.add_argument('--include-rses', action="store", default=None, type=str,
+                        help='RSE expression to include')
+    parser.add_argument('--rses', nargs='+', type=str,
+                        help='Explicit list of RSEs to include. If empty, it considers all RSEs')
+    parser.add_argument('--vos', nargs='+', type=str,
+                        help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument("--delay-seconds", action="store", default=600, type=int,
+                        help='The delay (seconds) to query replicas in BEING_DELETED state.')
+    parser.add_argument('--scheme', action="store", default=None, type=str,
+                        help='Force the reaper to use a particular protocol/scheme, e.g., mock')
+    parser.add_argument("--auto-exclude-threshold", action="store", default=100, type=int,
+                        help='Number of service unavailable exceptions after which the RSE gets temporarily excluded.')
+    parser.add_argument("--auto-exclude-timeout", action="store", default=600, type=int,
+                        help='Timeout for temporarily excluded RSEs.')
+
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(threads=args.threads,
+            chunk_size=args.chunk_size,
+            include_rses=args.include_rses,
+            exclude_rses=args.exclude_rses,
+            rses=args.rses,
+            vos=args.vos,
+            once=args.run_once,
+            greedy=args.greedy,
+            scheme=args.scheme,
+            delay_seconds=args.delay_seconds,
+            sleep_time=args.sleep_time,
+            auto_exclude_threshold=args.auto_exclude_threshold,
+            auto_exclude_timeout=args.auto_exclude_timeout)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-replica-recoverer

@@ -0,0 +1,248 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Replica-Recoverer is a daemon that declares suspicious replicas as bad if they are found available on other RSE.
+Consequently, automatic replica recovery is triggered via necromancer daemon, which creates a rule for such bad replicas.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.replicarecoverer.suspicious_replica_recoverer import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description='''
+Replica-Recoverer is a daemon that declares suspicious replicas
+that are available on other RSE as bad. Consequently, automatic
+replica recovery is triggered via necromancer daemon,
+which creates a rule for such bad replicas.''', epilog='''
+
+Preparing RSEs and DIDs for testing this daemon from rucio docker:
+-------------------------------------------------------------------
+
+$ sudo docker exec -it dev_rucio_1 /bin/bash
+
+Adding the RSEs to rucio DB:
+
+$ rucio-admin rse add MOCK_SUSPICIOUS
+$ rucio-admin rse set-attribute --rse MOCK_SUSPICIOUS --key backend_type --value file
+$ rucio-admin rse set-attribute --rse MOCK_SUSPICIOUS --key storage_usage_tool --value 'rucio.rse.protocols.posix.Default.getSpace'
+$ rucio-admin rse add-protocol --hostname localhost --scheme file --prefix '/tmp/rucio_rse1/' --space-token 'ATLASDATADISK1' --web-service-path '/srm/managerv2?SFN=' --domain-json '{ "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1}}' --impl 'rucio.rse.protocols.posix.Default'  MOCK_SUSPICIOUS
+$ rucio-admin rse info MOCK_SUSPICIOUS
+
+$ rucio-admin rse add MOCK_RECOVERY
+$ rucio-admin rse set-attribute --rse MOCK_RECOVERY --key backend_type --value POSIX
+$ rucio-admin rse set-attribute --rse MOCK_RECOVERY --key storage_usage_tool --value 'rucio.rse.protocols.posix.Default.getSpace'
+$ rucio-admin rse add-protocol --hostname localhost --scheme file --prefix '/tmp/rucio_rse2/' --space-token 'ATLASDATADISK2' --web-service-path '/srm/managerv2?SFN=' --domain-json '{ "lan": { "read": 1, "write": 1, "delete": 1 }, "wan": { "read": 1, "write": 1, "delete": 1}}' --impl 'rucio.rse.protocols.posix.Default'  MOCK_RECOVERY
+$ rucio-admin rse info MOCK_RECOVERY
+
+For testing, we create the following files:
+-------------------------------------------------------------------
+
+Name                                  MOCK_RECOVERY       MOCK_SUSPICIOUS    BAD (on MOCK_SUSPICIOUS)    SUSPICIOUS (on MOCK_SUSPICIOUS)
+file_available_suspicious             yes                 yes                no                          yes
+file_available_suspicious_and_bad     yes                 yes                yes                         yes
+file_notavailable_suspicious          unavailable         yes                no                          yes
+
+Only file_available_suspicious should be the one on which the daemon takes action and declares it as bad.
+
+$ id0=`uuidgen`
+$ id1=`uuidgen`
+$ id2=`uuidgen`
+$ id3=`uuidgen`
+$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS  (11 times)" > /tmp/file_available_suspicious'_'$id1
+$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS  (11 times) and 1 time bad/deleted/lost on MOCK_SUSPICIOUS" > /tmp/file_available_suspicious_and_bad'_'$id2
+$ echo "file declared as unavailable on MOCK_RECOVERY and declared as suspicious 11 times on MOCK_SUSPICIOUS" > /tmp/file_notavailable_suspicious'_'$id3
+
+Uploading the files created above to rucio:
+-------------------------------------------------------------------
+
+rucio add-dataset mock:dataset_of_suspicious_replicas'_'$id0
+
+Added mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778
+
+$ rucio add-rule mock:dataset_of_suspicious_replicas'_'$id0 1 MOCK_SUSPICIOUS
+$ rucio add-rule --source-replica-expression MOCK_SUSPICIOUS mock:dataset_of_suspicious_replicas'_'$id0 1 MOCK_RECOVERY
+$ rucio list-rules mock:dataset_of_suspicious_replicas'_'$id0
+
+ID                                ACCOUNT    SCOPE:NAME                                                                STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
+--------------------------------  ---------  ------------------------------------------------------------------------  ----------------------  ----------------  --------  ---------------  -------------------
+2a1a078b66ca4e209cc20a5826125334  root       mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778  OK[0/0/0]               MOCK_RECOVERY            1                   2019-02-19 14:12:30
+8c15d2f8e94a459a86a488055a10d068  root       mock:dataset_of_suspicious_replicas_2ba45524-860b-43f9-a601-6ccec2c46778  OK[0/0/0]               MOCK_SUSPICIOUS          1                   2019-02-19 14:12:28
+
+$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_available_suspicious'_'$id1 /tmp/file_available_suspicious'_'$id1
+$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_available_suspicious_and_bad'_'$id2 /tmp/file_available_suspicious_and_bad'_'$id2
+$ rucio upload --scope mock --rse MOCK_SUSPICIOUS --name file_notavailable_suspicious'_'$id3 /tmp/file_notavailable_suspicious'_'$id3
+$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_available_suspicious'_'$id1 /tmp/file_available_suspicious'_'$id1
+$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_available_suspicious_and_bad'_'$id2 /tmp/file_available_suspicious_and_bad'_'$id2
+$ rucio upload --scope mock --rse MOCK_RECOVERY --name file_notavailable_suspicious'_'$id3 /tmp/file_notavailable_suspicious'_'$id3
+[...]
+$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_available_suspicious'_'$id1
+$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_available_suspicious_and_bad'_'$id2
+$ rucio attach mock:dataset_of_suspicious_replicas'_'$id0 mock:file_notavailable_suspicious'_'$id3
+[...]
+
+$ rucio list-file-replicas mock:file_available_suspicious'_'$id1
++---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------+
+| SCOPE   | NAME                                                           | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                 |
+|---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------|
+| mock    | file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e | 87.000 B   | 206b1c91  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e |
+| mock    | file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e | 87.000 B   | 206b1c91  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e   |
++---------+----------------------------------------------------------------+------------+-----------+------------------------------------------------------------------------------------------------------------------------------+
+
+$ rucio list-file-replicas mock:file_available_suspicious_and_bad'_'$id2
++---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------+
+| SCOPE   | NAME                                                                   | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                         |
+|---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------|
+| mock    | file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 | 134.000 B  | dfdf2bff  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 |
+| mock    | file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835 | 134.000 B  | dfdf2bff  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835   |
++---------+------------------------------------------------------------------------+------------+-----------+--------------------------------------------------------------------------------------------------------------------------------------+
+
+$ rucio list-file-replicas mock:file_notavailable_suspicious'_'$id3
++---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------+
+| SCOPE   | NAME                                                              | FILESIZE   | ADLER32   | RSE: REPLICA                                                                                                                    |
+|---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------|
+| mock    | file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f | 101.000 B  | 0c14223f  | MOCK_SUSPICIOUS: file://localhost:0/tmp/rucio_rse1/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f |
+| mock    | file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f | 101.000 B  | 0c14223f  | MOCK_RECOVERY: file://localhost:0/tmp/rucio_rse2/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f   |
++---------+-------------------------------------------------------------------+------------+-----------+---------------------------------------------------------------------------------------------------------------------------------+
+
+Modifying the file statuses in the DB:
+--------------------------------------
+
+$ python
+
+# the paths below point to MOCK_SUSPICIOUS RSE (.../rucio_rse1)
+
+$$ file1 = ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e',]
+$$ file2 = ['file://localhost:0/tmp/rucio_rse1/mock/1d/2d/file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835',]
+$$ file3 = ['file://localhost:0/tmp/rucio_rse1/mock/7d/a6/file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f' ]
+
+$$ from rucio.client.replicaclient import ReplicaClient
+$$ replica_client = ReplicaClient()
+$$ import time
+$$ for i in range(11):
+       replica_client.declare_suspicious_file_replicas(file1, 'This is a good reason')
+       replica_client.declare_suspicious_file_replicas(file2, 'This is a good reason')
+       replica_client.declare_suspicious_file_replicas(file3, 'This is a good reason')
+       time.sleep(1)
+
+#  Declaring file2 bad on MOCK_SUSPICIOUS:
+   ---------------------------------------
+
+$$ replica_client.declare_bad_file_replicas(file2, 'This is a good reason')
+
+#  Update replica state of 'file_notavailable_suspicious'_'$id1' on MOCK_RECOVERY to 'UNAVAILABLE'
+#  (change the file name below according to the info from rucio !):
+   -----------------------------------------------------------------------------------------------
+
+$$ replica_client.update_replicas_states('MOCK_RECOVERY', [{'scope':'mock', 'name':'file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f', 'state':'U'}])
+
+
+# Checking the results of the file status changes:
+  ------------------------------------------------
+$$ from rucio.core.replica import get_suspicious_files
+$$ from datetime import datetime, timedelta
+$$ from_date = datetime.now() - timedelta(days=3)
+$$ from rucio.core.replica import list_bad_replicas_status
+
+$$ get_suspicious_files('MOCK_SUSPICIOUS',from_date,10)
+
+[{'created_at': datetime.datetime(2019, 2, 19, 14, 12, 56), 'scope': 'mock', 'cnt': 11L, 'name': 'file_notavailable_suspicious_6157f589-80db-492c-acdd-ef5f0c45112f', 'rse': 'MOCK_SUSPICIOUS'},
+ {'created_at': datetime.datetime(2019, 2, 19, 14, 12, 48), 'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
+
+$$ list_bad_replicas_status(rse='MOCK_SUSPICIOUS', younger_than=from_date)
+
+[{'name': 'file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835', 'rse': 'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'updated_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'state': BAD, 'scope': 'mock'}]
+
+$$ exit()
+
+Run the daemon:
+---------------
+
+  $ python bin/rucio-replica-recoverer --run-once --rse-expression='MOCK_SUSPICIOUS'
+
+Terminal output:
+----------------
+
+2019-02-19 14:39:24,114 709 INFO  replica_recoverer[0/0]: ready to query replicas at RSEs like *MOCK*, declared as suspicious in the last 3 days at least 10 times and which are available on other RSEs.
+2019-02-19 14:39:24,124 709 INFO  replica_recoverer[0/0]: suspicious replica query took 0.0101511478424 seconds, total of 1 replicas were found. [{'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
+2019-02-19 14:39:24,125 709 INFO  replica_recoverer[0/0]: looking for replica pfns.
+2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: found 1/1 pfns (took 0.035572052002 seconds) - declaring them as bad replicas now.
+2019-02-19 14:39:24,160 709 INFO  replica_recoverer[0/0]: ready to declare 1 bad replica(s) on MOCK_SUSPICIOUS: ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e'].
+2019-02-19 14:39:24,188 709 INFO  replica_recoverer[0/0]: finished declaring bad replicas on MOCK_SUSPICIOUS.
+2019-02-19 14:39:24,192 709 INFO  replica_recoverer[0/0]: graceful stop done
+
+
+# Checking the results of the file status changes:
+  ------------------------------------------------
+$ python
+
+$$ from rucio.core.replica import get_suspicious_files
+$$ from datetime import datetime, timedelta
+$$ from_date = datetime.now() - timedelta(days=3)
+$$ from rucio.core.replica import list_bad_replicas_status
+$$ from rucio.common.constants import SuspiciousAvailability
+
+$$ get_suspicious_files('MOCK_SUSPICIOUS',younger_than=from_date, nattempts=10, is_suspicious=True, available_elsewhere=SuspiciousAvailability["EXIST_COPIES"].value)
+
+>>> get_suspicious_files('MOCK_SUSPICIOUS',younger_than=from_date, nattempts=10, is_suspicious=True, available_elsewhere=SuspiciousAvailability["EXIST_COPIES"].value)
+[]
+
+$$ list_bad_replicas_status(rse='MOCK_SUSPICIOUS', younger_than=from_date)
+
+[{'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 39, 24), 'updated_at': datetime.datetime(2019, 2, 19, 14, 39, 24), 'state': BAD, 'scope': 'mock'},
+ {'name': 'file_available_suspicious_and_bad_46964411-95d4-46c4-a973-72c045195835', 'rse': 'MOCK_SUSPICIOUS', 'created_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'updated_at': datetime.datetime(2019, 2, 19, 14, 18, 33), 'state': BAD, 'scope': 'mock'}]
+
+$$ exit()
+
+When run in multi-VO mode, by default the daemon will run on RSEs from all VOs::
+
+  $ rucio-replica-recoverer --run-once
+  2020-07-28 15:15:14,151 5461    INFO    replica_recoverer: This instance will work on VOs: def, abc, xyz, 123
+
+By using the ``--vos`` argument only the VO or VOs specified will be affected::
+
+  $ rucio-replica-recoverer --run-once --vos abc xyz
+  2020-07-28 15:16:36,066 5474    INFO    replica_recoverer: This instance will work on VOs: abc, xyz
+
+Note that attempting the use the ``--vos`` argument when in single-VO mode will have no affect::
+
+  $ rucio-replica-recoverer --run-once --vos abc xyz
+  2020-07-28 15:21:33,349 5488    WARNING Ignoring argument vos, this is only applicable in a multi-VO setup.
+''', formatter_class=argparse.RawDescriptionHelpFormatter)  # NOQA: E501
+    parser.add_argument("--nattempts", action="store", default=5, type=int, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 5.')
+    parser.add_argument("--younger-than", action="store", default=5, type=int, help='Consider all file replicas logged in bad_replicas table since specified number of younger-than days. Default value is 5.')
+    parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
+    parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, type=int, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
+    parser.add_argument('--json-file-name', action="store", default="/opt/rucio/etc/suspicious_replica_recoverer.json", type=str, help='Name of the json file that that contains the policies which will be used by the suspicious replica recoverer.')
+    parser.add_argument('--sleep-time', action="store", default=3600, type=int, help='Concurrency control: Thread sleep time after each chunk of work.')
+    parser.add_argument('--active-mode', action="store_true", default=False, help='If NOT specified, the daemon will run without taking any actions on any files. In either case, the log file will be produced normally.')
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    PARSER = get_parser()
+    ARGS = PARSER.parse_args()
+    try:
+        run(once=ARGS.run_once, younger_than=ARGS.younger_than, nattempts=ARGS.nattempts, vos=ARGS.vos, limit_suspicious_files_on_rse=ARGS.limit_suspicious_files_on_rse, json_file_name=ARGS.json_file_name, sleep_time=ARGS.sleep_time, active_mode=ARGS.active_mode)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-rse-decommissioner

@@ -0,0 +1,66 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+RSE-Decommissioner is a daemon that clears RSEs scheduled for decommissioning.
+The actual operations (e.g. just delete all replicas, move the replicas to
+specific RSEs, etc.) that are performed on the RSEs depend on the
+"decommissioning profile", which must be specified for each RSE through RSE
+attributes.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.rsedecommissioner.rse_decommissioner import run, stop
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """Construct and return the argparse parser."""
+
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        epilog="""\
+To mark an RSE for decommissioning
+----------------------------------
+
+Clearing with the generic "delete all" profile:
+
+$ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
+              --value profile=generic_delete
+
+Clearing with the generic "move all" profile:
+
+$ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
+              --value profile=generic_move,destination=ANOTHER_RSE""",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument('--run-once', dest='run_once', action='store_true',
+                        help='One iteration only.')
+    parser.add_argument('--sleep-time', dest='sleep_time', action='store',
+                        default=86400, type=int,
+                        help='Concurrency control: thread sleep time after each'
+                        ' chunk of work')
+    return parser
+
+
+if __name__ == '__main__':
+    signal.signal(signal.SIGTERM, stop)
+    args = get_parser().parse_args()
+    try:
+        run(once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-storage-consistency-actions

@@ -0,0 +1,74 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Storage-Consistency-Actions is a daemon to trigger the
+deletion of dark files and re-transfer missing files at
+sites, as identified by the Storage-Consistency-Scanner.
+"""
+
+import argparse
+import pathlib
+import signal
+
+from rucio.daemons.storage.consistency.actions import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Consistency-Actions daemon is responsible for applying the corrective actions resulting from a consistency-check scan of an RSE.", epilog='''
+Run the daemon::
+  $ rucio-storage-consistency-actions --run-once --scope cms --rses T2_US_Purdue T2_US_Nebraska --dark-threshold-percent 2.0 --miss-threshold-percent 1.5 --scanner-files-path /tmp/consistency-dump --sleep-time 10
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--scope", action="store", default=False, type=str,
+                        help='Scope of the input files produced by theiCC scanner - e.g.: "cms" ')
+    parser.add_argument("--rses", nargs='+', default=[], type=str,
+                        help='RSEs to check, specified as a RSE expression. Defaults to check all RSEs.')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int,
+                        help='Concurrency control: thread sleep time (in seconds) after each chunk of work')
+    parser.add_argument('--dark-min-age', action="store", default=28, type=int,
+                        help='Min. age (in days) of a file to be considered as DARK.')
+    parser.add_argument('--dark-threshold-percent', action="store", default=1.0,
+                        type=float, help='Max. percentage of dark files at RSE, expressed as percents - e.g. 1.5 means 1.5%%')
+    parser.add_argument('--miss-threshold-percent', action="store", default=1.0,
+                        type=float, help='Max. percentage of missing files at RSE, expressed as percents - e.g. 1.5 means 1.5%%')
+    parser.add_argument('--scanner-files-path', action="store",
+                        default="/var/cache/consistency-dump", type=pathlib.Path,
+                        help='The path where the CC scanner files are mounted.')
+    parser.add_argument("--threads", action="store", default=1, type=int,
+                        help='Concurrency control: total number of threads on this process')
+    parser.add_argument('-f', '--force-proceed', action="store_true", default=False,
+                        help='Force CC actions, even if number of dark/missing files over thresholds')
+
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, scope=args.scope, rses=args.rses, sleep_time=args.sleep_time,
+            default_dark_min_age=args.dark_min_age,
+            default_dark_threshold_percent=args.dark_threshold_percent,
+            default_miss_threshold_percent=args.miss_threshold_percent,
+            force_proceed=args.force_proceed,
+            default_scanner_files_path=args.scanner_files_path,
+            threads=args.threads)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-transmogrifier

@@ -0,0 +1,77 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Transmogrifier (Replication Rule Resolver) Daemon
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.transmogrifier.transmogrifier import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(description="The Transmogrifier daemon is responsible for the creation of replication rules for DIDs matching a subscription.", epilog='''
+Create a DID::
+
+  $ python
+  from rucio.core.did import add_did
+  from rucio.db.sqla.constants import DIDType
+  add_did(scope='mock', name='test', type=DIDType.DATASET, account='root', meta={'project': 'test_project'})
+
+Create a subscription that matches the DID::
+
+  $ rucio-admin subscription add test '{"scope": ["mock"], "project": ["test_project"]}' '[{"copies": 1, "rse_expression": "MOCK", "activity": "User Subscriptions"}]' 'df'
+
+Check if there are rules for the DID::
+
+  $ rucio list-rules mock:test
+  ID                                ACCOUNT    SCOPE:NAME    STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
+  --------------------------------  ---------  ------------  ----------------------  ----------------  --------  ---------------  -------------------
+
+Run the daemon::
+
+  $ rucio-transmogrifier --run-once
+
+Check again if there are rules for the DID::
+
+  $ rucio list-rules mock:test
+  ID                                ACCOUNT    SCOPE:NAME    STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
+  --------------------------------  ---------  ------------  ----------------------  ----------------  --------  ---------------  -------------------
+  e658f6f47f444326aad624dabef7b785  root       mock:test     OK[0/0/0]               MOCK                     1                   2018-12-03 14:01:19
+    ''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Seconds to sleep between two cycles')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, threads=args.threads, bulk=args.bulk, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-37.0.0rc1.data/scripts/rucio-undertaker

@@ -0,0 +1,76 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Undertaker is a daemon to manage expired did.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.undertaker.undertaker import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Undertaker daemon is responsible for managing expired DIDs. It deletes DIDs, but not replicas by checking if there are DIDs where the 'expired_at' date property is older than the current timestamp.", epilog='''
+Create a DID that is already expired by setting its lifetime to -1::
+
+  $ python
+  from rucio.db.sqla.constants import DIDType
+  from rucio.client.didclient import DIDClient
+  client = DIDClient()
+  client.add_did(scope='mock', name='test', type=DIDType.DATASET, lifetime=-1)
+
+Check if the DID exists::
+
+  $ rucio list-dids mock:test
+  +--------------+--------------+
+  | SCOPE:NAME   | [DID TYPE]   |
+  |--------------+--------------|
+  | mock:test    | DATASET      |
+  +--------------+--------------+
+
+Run the daemon::
+
+  $ rucio-undertaker --run-once
+
+Check if the DID exists::
+
+  $ rucio list-dids mock:test
+  +--------------+--------------+
+  | SCOPE:NAME   | [DID TYPE]   |
+  |--------------+--------------|
+  +--------------+--------------+
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--total-workers", action="store", default=1, type=int, help='Total number of workers')
+    parser.add_argument("--chunk-size", action="store", default=5, type=int, help='Chunk size')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(total_workers=args.total_workers, chunk_size=args.chunk_size, once=args.run_once,
+            sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()