rucio 32.8.6__py3-none-any.whl

rucio/daemons/undertaker/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/daemons/undertaker/undertaker.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Undertaker is a daemon to manage expired did.
+'''
+
+import functools
+import logging
+import threading
+import traceback
+from copy import deepcopy
+from datetime import datetime, timedelta
+from random import randint
+from re import match
+from typing import TYPE_CHECKING
+
+from sqlalchemy.exc import DatabaseError
+
+import rucio.db.sqla.util
+from rucio.common.exception import DatabaseException, UnsupportedOperation, RuleNotFound
+from rucio.common.logging import setup_logging
+from rucio.common.types import InternalAccount
+from rucio.common.utils import chunks
+from rucio.core.did import list_expired_dids, delete_dids
+from rucio.core.monitor import MetricManager
+from rucio.daemons.common import run_daemon, HeartbeatHandler
+
+if TYPE_CHECKING:
+    from types import FrameType
+    from typing import Optional
+
+logging.getLogger("requests").setLevel(logging.CRITICAL)
+
+METRICS = MetricManager(module=__name__)
+graceful_stop = threading.Event()
+DAEMON_NAME = 'undertaker'
+
+
+def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10):
+    """
+    Main loop to select and delete dids.
+    """
+    paused_dids = {}  # {(scope, name): datetime}
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable=DAEMON_NAME,
+        partition_wait_time=1,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            paused_dids=paused_dids,
+            chunk_size=chunk_size,
+        )
+    )
+
+
+def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_handler: HeartbeatHandler, **_kwargs):
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    try:
+        # Refresh paused dids
+        iter_paused_dids = deepcopy(paused_dids)
+        for key in iter_paused_dids:
+            if datetime.utcnow() > paused_dids[key]:
+                del paused_dids[key]
+
+        dids = list_expired_dids(worker_number=worker_number, total_workers=total_workers, limit=10000)
+
+        dids = [did for did in dids if (did['scope'], did['name']) not in paused_dids]
+
+        if not dids:
+            logger(logging.INFO, 'did not get any work')
+            return
+
+        for chunk in chunks(dids, chunk_size):
+            _, _, logger = heartbeat_handler.live()
+            try:
+                logger(logging.INFO, 'Receive %s dids to delete', len(chunk))
+                delete_dids(dids=chunk, account=InternalAccount('root', vo='def'), expire_rules=True)
+                logger(logging.INFO, 'Delete %s dids', len(chunk))
+                METRICS.counter(name='undertaker.delete_dids').inc(len(chunk))
+            except RuleNotFound as error:
+                logger(logging.ERROR, error)
+            except (DatabaseException, DatabaseError, UnsupportedOperation) as e:
+                if match('.*ORA-00054.*', str(e.args[0])) or match('.*55P03.*', str(e.args[0])) or match('.*3572.*', str(e.args[0])):
+                    for did in chunk:
+                        paused_dids[(did['scope'], did['name'])] = datetime.utcnow() + timedelta(seconds=randint(600, 2400))
+                    METRICS.counter('delete_dids.exceptions.{exception}').labels(exception='LocksDetected').inc()
+                    logger(logging.WARNING, 'Locks detected for chunk')
+                else:
+                    logger(logging.ERROR, 'Got database error %s.', str(e))
+    except:
+        logging.critical(traceback.format_exc())
+
+
+def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) -> None:
+    """
+    Graceful exit.
+    """
+    graceful_stop.set()
+
+
+def run(once: bool = False, total_workers: int = 1, chunk_size: int = 10, sleep_time: int = 60):
+    """
+    Starts up the undertaker threads.
+    """
+    setup_logging(process_name=DAEMON_NAME)
+
+    if rucio.db.sqla.util.is_old_db():
+        raise DatabaseException('Database was not updated, daemon won\'t start')
+
+    if once:
+        undertaker(once)
+    else:
+        logging.info('main: starting threads')
+        threads = [threading.Thread(target=undertaker, kwargs={'once': once, 'chunk_size': chunk_size,
+                                                               'sleep_time': sleep_time}) for i in range(0, total_workers)]
+        [t.start() for t in threads]
+        logging.info('main: waiting for interrupts')
+
+        # Interruptible joins require a timeout.
+        while threads[0].is_alive():
+            [t.join(timeout=3.14) for t in threads]

rucio/db/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/db/sqla/__init__.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sqlalchemy.sql.expression import bindparam, text
+
+
+def filter_thread_work(session, query, total_threads, thread_id, hash_variable=None):
+    """ Filters a query to partition thread workloads based on the thread id and total number of threads """
+    if thread_id is not None and total_threads is not None and (total_threads - 1) > 0:
+        if session.bind.dialect.name == 'oracle':
+            bindparams = [bindparam('thread_id', thread_id), bindparam('total_threads', total_threads - 1)]
+            if not hash_variable:
+                query = query.filter(text('ORA_HASH(id, :total_threads) = :thread_id').bindparams(*bindparams))
+            else:
+                query = query.filter(text('ORA_HASH(%s, :total_threads) = :thread_id' % (hash_variable)).bindparams(*bindparams))
+        elif session.bind.dialect.name == 'mysql':
+            if not hash_variable:
+                query = query.filter(text('mod(md5(id), %s) = %s' % (total_threads, thread_id)))
+            else:
+                query = query.filter(text('mod(md5(%s), %s) = %s' % (hash_variable, total_threads, thread_id)))
+        elif session.bind.dialect.name == 'postgresql':
+            if not hash_variable:
+                query = query.filter(text('mod(abs((\'x\'||md5(id::text))::bit(32)::bigint), %s) = %s' % (total_threads, thread_id)))
+            else:
+                query = query.filter(text('mod(abs((\'x\'||md5(%s::text))::bit(32)::bigint), %s) = %s' % (hash_variable, total_threads, thread_id)))
+    return query

rucio/db/sqla/constants.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from enum import Enum
+
+# Individual constants
+
+OBSOLETE = datetime(year=1970, month=1, day=1)  # Tombstone value to mark obsolete replicas
+
+
+# The enum values below are the actual strings stored in the database -- these must be string types.
+# This is done explicitly via values_callable to SQLAlchemy enums in models.py and alembic scripts,
+# as overloading/overriding Python internal enums is discouraged.
+
+class AccountStatus(Enum):
+    ACTIVE = 'ACTIVE'
+    SUSPENDED = 'SUSPENDED'
+    DELETED = 'DELETED'
+
+
+class AccountType(Enum):
+    USER = 'USER'
+    GROUP = 'GROUP'
+    SERVICE = 'SERVICE'
+
+
+class BadFilesStatus(Enum):
+    BAD = 'B'
+    DELETED = 'D'
+    LOST = 'L'
+    RECOVERED = 'R'
+    SUSPICIOUS = 'S'
+    TEMPORARY_UNAVAILABLE = 'T'
+
+
+class BadPFNStatus(Enum):
+    BAD = 'B'
+    SUSPICIOUS = 'S'
+    TEMPORARY_UNAVAILABLE = 'T'
+    AVAILABLE = 'A'
+
+
+class DIDAvailability(Enum):
+    LOST = 'L'
+    DELETED = 'D'
+    AVAILABLE = 'A'
+
+
+class DIDReEvaluation(Enum):
+    ATTACH = 'A'
+    DETACH = 'D'
+
+
+class DIDType(Enum):
+    FILE = 'F'
+    DATASET = 'D'
+    CONTAINER = 'C'
+    ARCHIVE = 'A'
+    DELETED_FILE = 'X'
+    DELETED_DATASET = 'Y'
+    DELETED_CONTAINER = 'Z'
+
+
+class IdentityType(Enum):
+    X509 = 'X509'
+    GSS = 'GSS'
+    USERPASS = 'USERPASS'
+    SSH = 'SSH'
+    SAML = 'SAML'
+    OIDC = 'OIDC'
+
+
+class KeyType(Enum):
+    ALL = 'ALL'
+    COLLECTION = 'COLLECTION'
+    CONTAINER = 'CONTAINER'
+    DATASET = 'DATASET'
+    FILE = 'FILE'
+    DERIVED = 'DERIVED'
+
+
+class LifetimeExceptionsState(Enum):
+    APPROVED = 'A'
+    REJECTED = 'R'
+    WAITING = 'W'
+
+
+class LockState(Enum):
+    REPLICATING = 'R'
+    OK = 'O'
+    STUCK = 'S'
+
+
+class ReplicaState(Enum):
+    # Update rucio.common.constants at the same time than this
+    AVAILABLE = 'A'
+    UNAVAILABLE = 'U'
+    COPYING = 'C'
+    BEING_DELETED = 'B'
+    BAD = 'D'
+    TEMPORARY_UNAVAILABLE = 'T'
+
+
+class RequestErrMsg(Enum):
+    NO_SOURCES = 'NO_SOURCES'
+    SUBMISSION_FAILED = 'SUBMISSION_FAILED'
+    TRANSFER_FAILED = 'TRANSFER_FAILED'
+    MISMATCH_SCHEME = 'MISMATCH_SCHEME'
+    OTHER = 'OTHER'
+
+
+class RequestState(Enum):
+    QUEUED = 'Q'
+    SUBMITTING = 'G'
+    SUBMITTED = 'S'
+    FAILED = 'F'
+    DONE = 'D'
+    LOST = 'L'
+    NO_SOURCES = 'N'
+    ONLY_TAPE_SOURCES = 'O'
+    SUBMISSION_FAILED = 'A'
+    MISMATCH_SCHEME = 'M'
+    SUSPEND = 'U'
+    WAITING = 'W'
+    PREPARING = 'P'
+
+
+class RequestType(Enum):
+    TRANSFER = 'T'
+    UPLOAD = 'U'
+    DOWNLOAD = 'D'
+    STAGEIN = 'I'
+    STAGEOUT = 'O'
+
+
+class RSEType(Enum):
+    DISK = 'DISK'
+    TAPE = 'TAPE'
+
+
+class RuleGrouping(Enum):
+    ALL = 'A'
+    DATASET = 'D'
+    NONE = 'N'
+
+
+class RuleNotification(Enum):
+    YES = 'Y'
+    NO = 'N'
+    CLOSE = 'C'
+    PROGRESS = 'P'
+
+
+class RuleState(Enum):
+    REPLICATING = 'R'
+    OK = 'O'
+    STUCK = 'S'
+    SUSPENDED = 'U'
+    WAITING_APPROVAL = 'W'
+    INJECT = 'I'
+
+
+class ScopeStatus(Enum):
+    OPEN = 'O'
+    CLOSED = 'C'
+    DELETED = 'D'
+
+
+class SubscriptionState(Enum):
+    ACTIVE = 'A'
+    INACTIVE = 'I'
+    NEW = 'N'
+    UPDATED = 'U'
+    BROKEN = 'B'
+
+
+class TransferLimitDirection(Enum):
+    SOURCE = 'S'
+    DESTINATION = 'D'

rucio/db/sqla/migrate_repo/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/db/sqla/migrate_repo/env.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """
+    Run migrations in 'offline' mode.
+
+    This configures the context with just a URL/dialect
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+    """
+
+    # try getting url & version_table_schema
+    url = config.get_main_option("sqlalchemy.url")
+    version_table_schema = config.get_main_option("version_table_schema")
+
+    # try getting dialect in case URL has not been specified
+    dialect = config.get_main_option("dialect")
+
+    # try getting starting revision
+    starting_rev = config.get_main_option("starting_rev")
+
+    context.configure(
+        url=url,
+        version_table_schema=version_table_schema,
+        dialect_name=dialect,
+        starting_rev=starting_rev,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        include_schemas=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """
+    Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+    """
+
+    params = config.get_section(config.config_ini_section)
+
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        # Forcing the default is needed for PostgreSQL installations with named schemas.
+        # For other databases it doesn't matter.
+        # https://github.com/sqlalchemy/alembic/issues/409
+        conn = connection.execution_options(schema_translate_map={None: params.get('version_table_schema', None)})
+
+        context.configure(
+            connection=conn,
+            target_metadata=target_metadata,
+            version_table_schema=params.get('version_table_schema', None),
+            include_schemas=True)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+''' add new rule notification state progress '''
+
+from alembic import context, op
+from alembic.op import create_check_constraint
+
+from rucio.db.sqla.util import try_drop_constraint
+
+# Alembic revision identifiers
+revision = '01eaf73ab656'
+down_revision = '90f47792bb76'
+
+
+def upgrade():
+    '''
+    Upgrade the database to this revision
+    '''
+
+    schema = context.get_context().version_table_schema + '.' if context.get_context().version_table_schema else ''
+
+    if context.get_context().dialect.name == 'oracle':
+        try_drop_constraint('RULES_NOTIFICATION_CHK', 'rules')
+        create_check_constraint(constraint_name='RULES_NOTIFICATION_CHK', table_name='rules',
+                                condition="notification in ('Y', 'N', 'C', 'P')")
+
+    elif context.get_context().dialect.name == 'postgresql':
+        op.execute('ALTER TABLE ' + schema + 'rules DROP CONSTRAINT IF EXISTS "RULES_NOTIFICATION_CHK", ALTER COLUMN notification TYPE CHAR')
+        op.execute('DROP TYPE \"RULES_NOTIFICATION_CHK\"')
+        op.execute("CREATE TYPE \"RULES_NOTIFICATION_CHK\" AS ENUM('Y', 'N', 'C', 'P')")
+        op.execute("ALTER TABLE %srules ALTER COLUMN notification TYPE \"RULES_NOTIFICATION_CHK\" USING notification::\"RULES_NOTIFICATION_CHK\"" % schema)
+
+    elif context.get_context().dialect.name == 'mysql':
+        create_check_constraint(constraint_name='RULES_NOTIFICATION_CHK', table_name='rules',
+                                condition="notification in ('Y', 'N', 'C', 'P')")
+
+
+def downgrade():
+    '''
+    Downgrade the database to the previous revision
+    '''
+
+    schema = context.get_context().version_table_schema + '.' if context.get_context().version_table_schema else ''
+
+    if context.get_context().dialect.name == 'oracle':
+        try_drop_constraint('RULES_NOTIFICATION_CHK', 'rules')
+        create_check_constraint(constraint_name='RULES_NOTIFICATION_CHK', table_name='rules',
+                                condition="notification in ('Y', 'N', 'C')")
+
+    elif context.get_context().dialect.name == 'postgresql':
+        op.execute('ALTER TABLE ' + schema + 'rules DROP CONSTRAINT IF EXISTS "RULES_NOTIFICATION_CHK", ALTER COLUMN notification TYPE CHAR')
+        op.execute('DROP TYPE "RULES_NOTIFICATION_CHK"')
+        op.execute("CREATE TYPE \"RULES_NOTIFICATION_CHK\" AS ENUM('Y', 'N', 'C')")
+        op.execute("ALTER TABLE %srules ALTER COLUMN notification TYPE \"RULES_NOTIFICATION_CHK\" USING notification::\"RULES_NOTIFICATION_CHK\"" % schema)
+
+    elif context.get_context().dialect.name == 'mysql':
+        create_check_constraint(constraint_name='RULES_NOTIFICATION_CHK', table_name='rules',
+                                condition="notification in ('Y', 'N', 'C')")

rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+''' add eol_at in rules '''
+
+import sqlalchemy as sa
+
+from alembic import context
+from alembic.op import add_column, drop_column
+
+
+# Alembic revision identifiers
+revision = '0437a40dbfd1'
+down_revision = 'a5f6f6e928a7'
+
+
+def upgrade():
+    '''
+    Upgrade the database to this revision
+    '''
+
+    if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
+        schema = context.get_context().version_table_schema if context.get_context().version_table_schema else ''
+        add_column('rules', sa.Column('eol_at', sa.DateTime), schema=schema)
+        add_column('rules_hist_recent', sa.Column('eol_at', sa.DateTime), schema=schema)
+        add_column('rules_history', sa.Column('eol_at', sa.DateTime), schema=schema)
+
+
+def downgrade():
+    '''
+    Downgrade the database to the previous revision
+    '''
+
+    if context.get_context().dialect.name in ['oracle', 'mysql', 'postgresql']:
+        schema = context.get_context().version_table_schema if context.get_context().version_table_schema else ''
+        drop_column('rules', 'eol_at', schema=schema)
+        drop_column('rules_hist_recent', 'eol_at', schema=schema)
+        drop_column('rules_history', 'eol_at', schema=schema)