rucio 35.7.0__py3-none-any.whl

rucio-35.7.0.data/scripts/rucio-conveyor-finisher

@@ -0,0 +1,58 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.conveyor.finisher import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="Conveyor is a group of daemons to manage file transfers. The conveyor-finisher is the responsible to update Rucio internal state after the transfer has finished.")
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='One iteration only')
+    parser.add_argument("--total-threads", action="store", default=1, type=int,
+                        help='Concurrency control: total number of threads per process')
+    parser.add_argument("--db-bulk", action="store", default=1000, type=int,
+                        help='Bulk control: number of transfers per db query')
+    parser.add_argument("--bulk", action="store", default=100, type=int,
+                        help='Bulk control: number of requests')
+    parser.add_argument('--sleep-time', action="store", default=30, type=int,
+                        help='Seconds to sleep if few requests')
+    parser.add_argument('--activities', nargs='+', type=str,
+                        help='Explicit list of activities to include')
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once,
+            total_threads=args.total_threads,
+            sleep_time=args.sleep_time,
+            activities=args.activities,
+            db_bulk=args.db_bulk,
+            bulk=args.bulk)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-conveyor-poller

@@ -0,0 +1,66 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.conveyor.poller import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="Conveyor is a daemon to manage file transfers. The conveyor-poller is in charge of poll the transfer tool to check if the submitted transfers are already done.")
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='One iteration only')
+    parser.add_argument("--fts-bulk", action="store", default=100, type=int,
+                        help='Bulk control: number of transfers per FTS query')
+    parser.add_argument("--db-bulk", action="store", default=1000, type=int,
+                        help='Bulk control: number of transfers per db query')
+    parser.add_argument("--older-than", action="store", default=60, type=int,
+                        help='Poll control: older request more than this value to poll')
+    parser.add_argument('--sleep-time', action="store", default=300, type=int,
+                        help='Seconds to sleep if few requests')
+    parser.add_argument('--activities', nargs='+', type=str,
+                        help='Explicit list of activities to include')
+    parser.add_argument('--activity-shares', action='store', default=None, type=str,
+                        help='JSON-encoded string of an activity shares dictionary {"act_1": 0.2, "act_2": 0.4, ...}')
+    parser.add_argument('--total-threads', action="store", default=1, type=int,
+                        help='Concurrency control: total number of threads for this process')
+
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once,
+            fts_bulk=args.fts_bulk,
+            db_bulk=args.db_bulk,
+            older_than=args.older_than,
+            sleep_time=args.sleep_time,
+            activities=args.activities,
+            activity_shares=args.activity_shares,
+            total_threads=args.total_threads)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-conveyor-preparer

@@ -0,0 +1,37 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import signal
+
+from rucio.daemons.conveyor.preparer import run, stop
+
+
+def main(args):
+    signal.signal(signal.SIGTERM, stop)
+    try:
+        run(once=args.run_once, threads=args.threads, sleep_time=args.sleep_time, bulk=args.bulk, ignore_availability=args.ignore_availability)
+    except KeyboardInterrupt:
+        stop()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads on this process')
+    parser.add_argument("--sleep-time", action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    parser.add_argument("--bulk", action="store", default=100, type=int, help='Limit of requests per chunk')
+    parser.add_argument('--ignore-availability', action="store_true", default=False, help='If True, will also try to submit transfers having blocklisted RSEs as sources')
+    main(args=parser.parse_args())

rucio-35.7.0.data/scripts/rucio-conveyor-receiver

@@ -0,0 +1,43 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+import argparse
+import signal
+
+from rucio.daemons.conveyor.receiver import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="Conveyor is a daemon to manage file transfers. The conveyor-receiver is similar to conveyor-poller, but instead of poll the transfer tool, conveyor-receiver gets knowledge about the state of submitted transfers by listening to the transfer tool messages.")  # noqa E501
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--total-threads", action="store", default=1, type=int, help='Concurrency control: total number of threads per process')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, total_threads=args.total_threads)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-conveyor-stager

@@ -0,0 +1,76 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.conveyor.stager import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The conveyor-stager is responsible for issuing staging requests to the tape system.")
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='One iteration only')
+    parser.add_argument("--total-threads", action="store", default=1, type=int,
+                        help='Concurrency control: total number of threads per process')
+    parser.add_argument("--bulk", action="store", default=100, type=int,
+                        help='Bulk control: number of requests')
+    parser.add_argument("--group-bulk", action="store", default=1, type=int,
+                        help='Group control: number of requests per group')
+    parser.add_argument("--group-policy", action="store", default='rule', type=str,
+                        help='Group control: policy used to group. enum{rule, dest, src_dest, rule_src_dest}')
+    parser.add_argument('--source-strategy', action="store", default=None, type=str,
+                        help='Source strategy. Overload the strategy defined in config DB.')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str,
+                        help='RSE expression to exclude')
+    parser.add_argument('--include-rses', action="store", default=None, type=str,
+                        help='RSE expression to include')
+    parser.add_argument('--rses', nargs='+', type=str,
+                        help='Explicit list of RSEs to include')
+    parser.add_argument('--vos', nargs='+', type=str,
+                        help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument('--activities', nargs='+', type=str,
+                        help='Explicit list of activities to include')
+    parser.add_argument('--sleep-time', action="store", default=600, type=int,
+                        help='Seconds to sleep if few requests')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once,
+            total_threads=args.total_threads,
+            bulk=args.bulk,
+            group_bulk=args.group_bulk,
+            include_rses=args.include_rses,
+            exclude_rses=args.exclude_rses,
+            rses=args.rses,
+            vos=args.vos,
+            source_strategy=args.source_strategy,
+            activities=args.activities,
+            sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-conveyor-submitter

@@ -0,0 +1,139 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.conveyor.submitter import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Conveyor-Submitter daemon is responsible for managing non-tape file transfers. It prepares transfer jobs and submits them to the transfertool.", epilog='''
+Upload a file and create a replication rule::
+
+  $ rucio upload --scope mock --rse MOCK --name file filename.txt
+  $ rucio add-rule mock:file 1 MOCK2
+  $ rucio-admin rse add-distance MOCK2 MOCK --distance 1
+
+The rule should replicate the file from RSE MOCK to RSE MOCK2. Therefore a distance between these RSEs is needed.
+
+Check transfer requests for the DID::
+
+    $ python
+    from rucio.db.sqla import session,models
+    session.get_session().query(models.Request).filter_by(scope='mock', name='file').first()
+    # {'request_type': TRANSFER, 'state': QUEUED', ...}
+
+A queued request was created which can be picked uped by the Conveyor-Submiter daemon.
+
+Run the daemon::
+
+  $ rucio-conveyor-submitter --run-once
+
+Check again the transfer requests for the DID::
+
+    $ python
+    from rucio.db.sqla import session,models
+    session.get_session().query(models.Request).filter_by(scope='mock', name='file').first()
+    # {'request_type': TRANSFER, 'state': SUBMITTED', ...}
+
+A transfer request got created by executing the transfer. Depending on the transfer submission, the request state can be different. In this example the transfer got submitted successfully.
+
+When run in multi-VO mode, by default the daemon will run on RSEs from all VOs::
+
+  $ rucio-conveyor-submitter --run-once
+  2020-07-29 13:51:09,436 5784    INFO    This instance will work on VOs: def, abc, xyz, 123
+  2020-07-29 13:51:13,315 5784    INFO    RSE selection: automatic for relevant VOs
+  2020-07-29 13:51:13,316 5784    INFO    starting submitter threads
+
+By using the ``--vos`` argument only the VO or VOs specified will be affected::
+
+  $ rucio-conveyor-submitter --run-once --vos abc xyz
+  2020-07-29 13:51:09,436 5784    INFO    This instance will work on VOs: abc, xyz
+  2020-07-29 13:51:13,315 5784    INFO    RSE selection: automatic for relevant VOs
+  2020-07-29 13:51:13,316 5784    INFO    starting submitter threads
+
+Note that attempting the use the ``--vos`` argument when in single-VO mode will have no affect::
+
+  $ rucio-conveyor-submitter --run-once --vos abc xyz
+  2020-07-29 13:39:37,263 5752    INFO    RSE selection: automatic
+  2020-07-29 13:39:37,264 5752    INFO    starting submitter threads
+    ''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='One iteration only')
+    parser.add_argument("--total-threads", action="store", default=1, type=int,
+                        help='Concurrency control: total number of threads per process')
+    parser.add_argument("--bulk", action="store", default=100, type=int,
+                        help='Bulk control: number of requests')
+    parser.add_argument("--group-bulk", action="store", default=1, type=int,
+                        help='Group control: number of requests per group')
+    parser.add_argument("--group-policy", action="store", default='rule', type=str,
+                        help='Group control: policy used to group. enum{rule, dest, src_dest, rule_src_dest, activity_dest, activity_src_dest}')
+    parser.add_argument('--source-strategy', action="store", default=None, type=str,
+                        help='Source strategy. Overload the strategy defined in config DB.')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str,
+                        help='RSE expression to exclude')
+    parser.add_argument('--include-rses', action="store", default=None, type=str,
+                        help='RSE expression to include')
+    parser.add_argument('--rses', nargs='+', type=str,
+                        help='Explicit list of RSEs to include')
+    parser.add_argument('--vos', nargs='+', type=str,
+                        help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument('--activities', nargs='+', type=str,
+                        help='Explicit list of activities to include')
+    parser.add_argument('--exclude-activities', nargs='+', type=str,
+                        help='Explicit list of activities to exclude')
+    parser.add_argument('--ignore-availability', action="store_true", default=False,
+                        help='If True, will also try to submit transfers having blocklisted RSEs as sources')
+    parser.add_argument('--sleep-time', action="store", default=600, type=int,
+                        help='Seconds to sleep if few requests')
+    parser.add_argument('--max-sources', action="store", default=5, type=int,
+                        help='Maximum source replicas per multi-source FTS job')
+    parser.add_argument('--archive-timeout-override', action="store", default=None, type=int, metavar="INTEGER_SECONDS",
+                        help='Override the archive_timeout parameter for any transfers with it set (0 to unset)')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once,
+            bulk=args.bulk,
+            group_bulk=args.group_bulk,
+            group_policy=args.group_policy,
+            include_rses=args.include_rses,
+            exclude_rses=args.exclude_rses,
+            rses=args.rses,
+            vos=args.vos,
+            source_strategy=args.source_strategy,
+            activities=args.activities,
+            exclude_activities=args.exclude_activities,
+            ignore_availability=args.ignore_availability,
+            sleep_time=args.sleep_time,
+            max_sources=args.max_sources,
+            archive_timeout_override=args.archive_timeout_override,
+            total_threads=args.total_threads)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-conveyor-throttler

@@ -0,0 +1,104 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Conveyor is a daemon to manage file transfers.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.conveyor.throttler import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Conveyor-Throttler daemon is responsible for managing the internal queue of transfer requests. Depending on transfer limits of current and waiting transfers, it decides whether a transfer should be put in the queue or not.", epilog='''
+In this example, the transfer limit will be one transfer which means that there should be only one active transfer at the time.
+There will be two waiting transfer requests and no current active transfer.
+After running the daemon, there will be one transfer in the queue which can then be started.
+
+Setup the transfer limit::
+
+  $ python
+  from rucio.core.rse import set_rse_transfer_limits
+  set_rse_transfer_limits('MOCK2', 'User Subscriptions', max_transfers=1)
+  $ rucio-admin config set --section throttler --option 'User Subscriptions,MOCK2' --value 1
+
+If Rucio is running in multi-VO mode, then the RSE ID should be used in place of its name when setting the config::
+
+  $ rucio-admin rse info MOCK2
+  Settings:
+  =========
+    ...
+  Attributes:
+  ===========
+    ...
+  Protocols:
+  ==========
+    ...
+  Usage:
+  ======
+    ...
+    rse_id: 9c54c73cbd534450b2202a576f809f1f
+  $ rucio-admin config set --section throttler --option 'User Subscriptions,9c54c73cbd534450b2202a576f809f1f' --value 1
+
+Upload two files and create replication rules to the same RSE::
+
+  $ rucio upload --scope mock --rse MOCK --name file1 filename.txt
+  $ rucio add-rule mock:file 1 MOCK2
+  $ rucio upload --scope mock --rse MOCK --name file2 filename.txt
+  $ rucio add-rule mock:file2 1 MOCK2
+
+Check transfer requests::
+
+    $ python
+    from rucio.db.sqla import session,models
+    [request.to_dict() for request in session.get_session().query(models.Request).all()]
+    # [{'state': WAITING, ...}, {'state': WAITING, ...}
+
+Two transfer requests with the state 'WAITING' got created.
+
+Run the daemon::
+
+  $ rucio-conveyor-throttler --run-once
+
+Check transfer requests::
+
+    $ python
+    from rucio.db.sqla import session,models
+    [request.to_dict() for request in session.get_session().query(models.Request).all()]
+    # [{'state': WAITING, ...}, {'state': QUEUED, ...}
+
+Finally one of the transfer requests got put in the queue and can be picked up by the Conyevor-Submitter daemon to submit the transfer job to the transfertool.
+The other request will have to wait until one of the queued requests is done or until the transfer limit changes.
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False,
+                        help='One iteration only')
+    parser.add_argument('--sleep-time', action="store", default=600, type=int,
+                        help='Seconds to sleep if few requests')
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-dark-reaper

@@ -0,0 +1,53 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Reaper is a daemon to manage file on disk but not in Rucio(dark data) deletion
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.reaper.dark_reaper import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Dark-Reaper daemon is responsible for the deletion of quarantined replicas.")
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--total-workers", action="store", default=1, type=int, help='Total number of workers per process')
+    parser.add_argument("--chunk-size", action="store", default=10, type=int, help='Chunk size')
+    parser.add_argument("--scheme", action="store", default=None, type=str, help='Force the reaper to use a particular protocol, e.g., mock.')
+    parser.add_argument('--rses', nargs='+', type=str, help='List of RSEs')
+    parser.add_argument('--exclude-rses', action="store", default=None, type=str, help='RSEs expression to exclude RSEs')
+    parser.add_argument('--include-rses', action="store", default=None, type=str, help='RSEs expression to include RSEs')
+    parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == '__main__':
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(total_workers=args.total_workers, chunk_size=args.chunk_size,
+            once=args.run_once, scheme=args.scheme, rses=args.rses,
+            exclude_rses=args.exclude_rses, include_rses=args.include_rses, vos=args.vos,
+            sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()