rucio 35.7.0__py3-none-any.whl

rucio-35.7.0.data/scripts/rucio-storage-consistency-actions

@@ -0,0 +1,74 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Storage-Consistency-Actions is a daemon to trigger the
+deletion of dark files and re-transfer missing files at
+sites, as identified by the Storage-Consistency-Scanner.
+"""
+
+import argparse
+import pathlib
+import signal
+
+from rucio.daemons.storage.consistency.actions import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Consistency-Actions daemon is responsible for applying the corrective actions resulting from a consistency-check scan of an RSE.", epilog='''
+Run the daemon::
+  $ rucio-storage-consistency-actions --run-once --scope cms --rses T2_US_Purdue T2_US_Nebraska --dark-threshold-percent 2.0 --miss-threshold-percent 1.5 --scanner-files-path /tmp/consistency-dump --sleep-time 10
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--scope", action="store", default=False, type=str,
+                        help='Scope of the input files produced by theiCC scanner - e.g.: "cms" ')
+    parser.add_argument("--rses", nargs='+', default=[], type=str,
+                        help='RSEs to check, specified as a RSE expression. Defaults to check all RSEs.')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int,
+                        help='Concurrency control: thread sleep time (in seconds) after each chunk of work')
+    parser.add_argument('--dark-min-age', action="store", default=28, type=int,
+                        help='Min. age (in days) of a file to be considered as DARK.')
+    parser.add_argument('--dark-threshold-percent', action="store", default=1.0,
+                        type=float, help='Max. percentage of dark files at RSE, expressed as percents - e.g. 1.5 means 1.5%%')
+    parser.add_argument('--miss-threshold-percent', action="store", default=1.0,
+                        type=float, help='Max. percentage of missing files at RSE, expressed as percents - e.g. 1.5 means 1.5%%')
+    parser.add_argument('--scanner-files-path', action="store",
+                        default="/var/cache/consistency-dump", type=pathlib.Path,
+                        help='The path where the CC scanner files are mounted.')
+    parser.add_argument("--threads", action="store", default=1, type=int,
+                        help='Concurrency control: total number of threads on this process')
+    parser.add_argument('-f', '--force-proceed', action="store_true", default=False,
+                        help='Force CC actions, even if number of dark/missing files over thresholds')
+
+    return parser
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, scope=args.scope, rses=args.rses, sleep_time=args.sleep_time,
+            default_dark_min_age=args.dark_min_age,
+            default_dark_threshold_percent=args.dark_threshold_percent,
+            default_miss_threshold_percent=args.miss_threshold_percent,
+            force_proceed=args.force_proceed,
+            default_scanner_files_path=args.scanner_files_path,
+            threads=args.threads)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-transmogrifier

@@ -0,0 +1,77 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Transmogrifier (Replication Rule Resolver) Daemon
+'''
+
+import argparse
+import signal
+
+from rucio.daemons.transmogrifier.transmogrifier import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(description="The Transmogrifier daemon is responsible for the creation of replication rules for DIDs matching a subscription.", epilog='''
+Create a DID::
+
+  $ python
+  from rucio.core.did import add_did
+  from rucio.db.sqla.constants import DIDType
+  add_did(scope='mock', name='test', type=DIDType.DATASET, account='root', meta={'project': 'test_project'})
+
+Create a subscription that matches the DID::
+
+  $ rucio-admin subscription add test '{"scope": ["mock"], "project": ["test_project"]}' '[{"copies": 1, "rse_expression": "MOCK", "activity": "User Subscriptions"}]' 'df'
+
+Check if there are rules for the DID::
+
+  $ rucio list-rules mock:test
+  ID                                ACCOUNT    SCOPE:NAME    STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
+  --------------------------------  ---------  ------------  ----------------------  ----------------  --------  ---------------  -------------------
+
+Run the daemon::
+
+  $ rucio-transmogrifier --run-once
+
+Check again if there are rules for the DID::
+
+  $ rucio list-rules mock:test
+  ID                                ACCOUNT    SCOPE:NAME    STATE[OK/REPL/STUCK]    RSE_EXPRESSION      COPIES  EXPIRES (UTC)    CREATED (UTC)
+  --------------------------------  ---------  ------------  ----------------------  ----------------  --------  ---------------  -------------------
+  e658f6f47f444326aad624dabef7b785  root       mock:test     OK[0/0/0]               MOCK                     1                   2018-12-03 14:01:19
+    ''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='Runs one loop iteration')
+    parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: number of threads')
+    parser.add_argument("--bulk", action="store", default=1000, type=int, help='Bulk control: number of requests per cycle')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Seconds to sleep between two cycles')
+    return parser
+
+
+if __name__ == "__main__":
+
+    # Bind our callback to the SIGTERM signal and run the daemon:
+    signal.signal(signal.SIGTERM, stop)
+
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(once=args.run_once, threads=args.threads, bulk=args.bulk, sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.data/scripts/rucio-undertaker

@@ -0,0 +1,76 @@
+#!python
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Undertaker is a daemon to manage expired did.
+"""
+
+import argparse
+import signal
+
+from rucio.daemons.undertaker.undertaker import run, stop
+
+
+def get_parser():
+    """
+    Returns the argparse parser.
+    """
+    parser = argparse.ArgumentParser(description="The Undertaker daemon is responsible for managing expired DIDs. It deletes DIDs, but not replicas by checking if there are DIDs where the 'expired_at' date property is older than the current timestamp.", epilog='''
+Create a DID that is already expired by setting its lifetime to -1::
+
+  $ python
+  from rucio.db.sqla.constants import DIDType
+  from rucio.client.didclient import DIDClient
+  client = DIDClient()
+  client.add_did(scope='mock', name='test', type=DIDType.DATASET, lifetime=-1)
+
+Check if the DID exists::
+
+  $ rucio list-dids mock:test
+  +--------------+--------------+
+  | SCOPE:NAME   | [DID TYPE]   |
+  |--------------+--------------|
+  | mock:test    | DATASET      |
+  +--------------+--------------+
+
+Run the daemon::
+
+  $ rucio-undertaker --run-once
+
+Check if the DID exists::
+
+  $ rucio list-dids mock:test
+  +--------------+--------------+
+  | SCOPE:NAME   | [DID TYPE]   |
+  |--------------+--------------|
+  +--------------+--------------+
+''', formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
+    parser.add_argument("--total-workers", action="store", default=1, type=int, help='Total number of workers')
+    parser.add_argument("--chunk-size", action="store", default=5, type=int, help='Chunk size')
+    parser.add_argument('--sleep-time', action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work')
+    return parser
+
+
+if __name__ == "__main__":
+
+    signal.signal(signal.SIGTERM, stop)
+    parser = get_parser()
+    args = parser.parse_args()
+    try:
+        run(total_workers=args.total_workers, chunk_size=args.chunk_size, once=args.run_once,
+            sleep_time=args.sleep_time)
+    except KeyboardInterrupt:
+        stop()

rucio-35.7.0.dist-info/METADATA

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: rucio
+Version: 35.7.0
+Summary: Rucio Package
+Home-page: https://rucio.cern.ch/
+Author: Rucio
+Author-email: rucio-dev@cern.ch
+License: Apache License, Version 2.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Environment :: No Input/Output (Daemon)
+Requires-Python: >=3.9, <4
+License-File: LICENSE
+License-File: AUTHORS.rst
+Requires-Dist: alembic==1.13.2
+Requires-Dist: argcomplete==3.4.0
+Requires-Dist: boto3==1.34.142
+Requires-Dist: dogpile-cache==1.2.2
+Requires-Dist: flask==3.0.3
+Requires-Dist: geoip2==4.8.0
+Requires-Dist: google-auth==2.32.0
+Requires-Dist: jsonschema==4.23.0
+Requires-Dist: oic==1.7.0
+Requires-Dist: paramiko==3.4.0
+Requires-Dist: prometheus-client==0.20.0
+Requires-Dist: pymemcache==4.0.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-magic==0.4.27
+Requires-Dist: redis==5.0.7
+Requires-Dist: requests==2.32.3
+Requires-Dist: sqlalchemy==2.0.31
+Requires-Dist: statsd==4.0.1
+Requires-Dist: stomp-py==8.1.2
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: urllib3==1.26.19
+Provides-Extra: oracle
+Requires-Dist: cx-oracle==8.3.0; extra == "oracle"
+Provides-Extra: mongo
+Requires-Dist: pymongo==4.8.0; extra == "mongo"
+Provides-Extra: postgresql
+Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"
+Provides-Extra: mysql
+Requires-Dist: pymysql==1.1.1; extra == "mysql"
+Provides-Extra: kerberos
+Requires-Dist: kerberos==1.3.1; extra == "kerberos"
+Requires-Dist: pykerberos==1.2.4; extra == "kerberos"
+Requires-Dist: requests-kerberos==0.15.0; extra == "kerberos"
+Provides-Extra: globus
+Requires-Dist: globus-sdk==3.41.0; extra == "globus"
+Requires-Dist: pyyaml==6.0.1; extra == "globus"
+Provides-Extra: saml
+Requires-Dist: python3-saml==1.16.0; extra == "saml"
+Provides-Extra: dev
+Requires-Dist: pyyaml==6.0.1; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary