rucio 37.0.0rc1__py3-none-any.whl

rucio/core/config.py

@@ -0,0 +1,437 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any, Optional, TypeVar
+
+from dogpile.cache.api import NoValue
+from sqlalchemy import and_, delete, func, select, update
+
+from rucio.common.cache import CacheKey, MemcacheRegion
+from rucio.common.exception import ConfigNotFound
+from rucio.db.sqla import models
+from rucio.db.sqla.session import read_session, transactional_session
+
+T = TypeVar('T')
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from sqlalchemy.orm import Session
+
+
+REGION = MemcacheRegion(expiration_time=900)
+
+SECTIONS_CACHE_KEY = 'sections'
+
+
+@read_session
+def sections(
+        *,
+        use_cache: bool = True,
+        expiration_time: int = 900,
+        session: "Session"
+) -> list[str]:
+    """
+    Return a list of the sections available.
+
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
+    :param session: The database session in use.
+    :returns: ['section_name', ...]
+    """
+
+    all_sections = NoValue()
+    if use_cache:
+        all_sections = read_from_cache(SECTIONS_CACHE_KEY, expiration_time)
+    if isinstance(all_sections, NoValue):
+        stmt = select(
+            models.Config.section
+        ).distinct(
+        )
+        all_sections = list(session.execute(stmt).scalars().all())
+        write_to_cache(SECTIONS_CACHE_KEY, all_sections)
+
+    return all_sections
+
+
+@transactional_session
+def add_section(section: str, *, session: "Session") -> None:
+    """
+    Add a section to the configuration.
+    :param session: The database session in use.
+    :param section: The name of the section.
+    """
+
+    raise NotImplementedError('Irrelevant - sections cannot exist without options')
+
+
+@read_session
+def has_section(
+        section: str,
+        *,
+        use_cache: bool = True,
+        expiration_time: int = 900,
+        session: "Session"
+) -> bool:
+    """
+    Indicates whether the named section is present in the configuration.
+
+    :param section: The name of the section.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
+    :param session: The database session in use.
+    :returns: True/False
+    """
+    has_section_key = 'has_section_%s' % section
+    has_section = NoValue()
+    if use_cache:
+        has_section = read_from_cache(has_section_key, expiration_time)
+    if isinstance(has_section, NoValue):
+        stmt = select(
+            models.Config
+        ).where(
+            models.Config.section == section
+        )
+        has_section = session.execute(stmt).first() is not None
+        write_to_cache(has_section_key, has_section)
+    return has_section
+
+
+@read_session
+def options(
+        section: str,
+        *,
+        use_cache: bool = True,
+        expiration_time: int = 900,
+        session: "Session"
+) -> list[str]:
+    """
+    Returns a list of options available in the specified section.
+
+    :param section: The name of the section.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
+    :param session: The database session in use.
+    :returns: ['option', ...]
+    """
+    options_key = CacheKey.options(section)
+    options = NoValue()
+    if use_cache:
+        options = read_from_cache(options_key, expiration_time)
+    if isinstance(options, NoValue):
+        stmt = select(
+            models.Config.opt
+        ).where(
+            models.Config.section == section
+        ).distinct()
+        options = list(session.execute(stmt).scalars().all())
+        write_to_cache(options_key, options)
+    return options
+
+
+@read_session
+def has_option(
+        section: str,
+        option: str,
+        *,
+        use_cache: bool = True,
+        expiration_time: int = 900,
+        session: "Session"
+) -> bool:
+    """
+    Check if the given section exists and contains the given option.
+
+    :param section: The name of the section.
+    :param option: The name of the option.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
+    :param session: The database session in use.
+    :returns: True/False
+    """
+    has_option_key = CacheKey.has_option(section, option)
+    has_option = NoValue()
+    if use_cache:
+        has_option = read_from_cache(has_option_key, expiration_time)
+    if isinstance(has_option, NoValue):
+        stmt = select(
+            models.Config
+        ).where(
+            and_(models.Config.section == section,
+                 models.Config.opt == option)
+        )
+        has_option = session.execute(stmt).first() is not None
+        write_to_cache(has_option_key, has_option)
+    return has_option
+
+
+@read_session
+def get(
+        section: str,
+        option: str,
+        *,
+        default: Optional[T] = None,
+        use_cache: bool = True,
+        expiration_time: int = 900,
+        convert_type_fnc: 'Callable[[str], T]',
+        session: "Session"
+) -> T:
+    """
+    Get an option value for the named section. Value can be auto-coerced to string, int, float, bool, None.
+
+    Caveat emptor: Strings, regardless the case, matching 'on'/off', 'true'/'false', 'yes'/'no' are converted to bool.
+                   0/1 are converted to int, and not to bool.
+
+    :param section: The name of the section.
+    :param option: The name of the option.
+    :param default: The default value if no value is found.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
+    :param convert_type_fnc: A function used to parse the string config value into the desired destination type
+    :param session: The database session in use.
+    :returns: The auto-coerced value.
+    """
+    value_key = CacheKey.value(section, option)
+    value = NoValue()
+    if use_cache:
+        value = read_from_cache(value_key, expiration_time)
+    if isinstance(value, NoValue):
+        stmt = select(
+            models.Config.value
+        ).where(
+            and_(models.Config.section == section,
+                 models.Config.opt == option)
+        )
+        tmp = session.execute(stmt).first()
+        if tmp is not None:
+            value = convert_type_fnc(tmp[0])
+            write_to_cache(value_key, tmp[0])
+        elif default is None:
+            raise ConfigNotFound
+        else:
+            value = default
+            write_to_cache(value_key, value)  # Also write default to cache
+    else:
+        value = convert_type_fnc(value)
+    return value
+
+
+@read_session
+def items(
+        section: str,
+        use_cache: bool = True,
+        expiration_time: int = 900,
+        *,
+        convert_type_fnc: 'Callable[[str], T]',
+        session: "Session"
+) -> list[tuple[str, T]]:
+    """
+    Return a list of (option, value) pairs for each option in the given section. Values are auto-coerced as in get().
+
+    :param section: The name of the section.
+    :param use_cache: Boolean if the cache should be used.
+    :param expiration_time: Time after that the cached value gets ignored.
+    :param convert_type_fnc: A function used to parse the string config value into the desired destination type
+    :param session: The database session in use.
+    :returns: [('option', auto-coerced value), ...]
+    """
+    items_key = CacheKey.items(section)
+    items = NoValue()
+    if use_cache:
+        items = read_from_cache(items_key, expiration_time)
+    if isinstance(items, NoValue):
+        stmt = select(
+            models.Config.opt,
+            models.Config.value
+        ).where(
+            models.Config.section == section
+        )
+        items = session.execute(stmt).all()
+        write_to_cache(items_key, items)
+    return [(opt, convert_type_fnc(val)) for opt, val in items]
+
+
+@transactional_session
+def set(
+        section: str,
+        option: str,
+        value: Any,
+        *,
+        session: "Session"
+) -> None:
+    """
+    Set the given option to the specified value. If the option doesn't exist, it is created.
+
+    :param section: The name of the section.
+    :param option: The name of the option.
+    :param value: The content of the value.
+    :param session: The database session in use.
+    """
+
+    if not has_option(section=section, option=option, use_cache=False, session=session):
+        section_existed = has_section(section=section)
+
+        new_option = models.Config(section=section, opt=option, value=value)
+        new_option.save(session=session)
+
+        delete_from_cache(key=CacheKey.value(section, option))
+        delete_from_cache(key=CacheKey.has_option(section, option))
+        delete_from_cache(key=CacheKey.items(section))
+        if not section_existed:
+            delete_from_cache(key=SECTIONS_CACHE_KEY)
+            delete_from_cache(key=CacheKey.has_section(section))
+    else:
+        stmt = select(
+            models.Config.value
+        ).where(
+            and_(models.Config.section == section,
+                 models.Config.opt == option)
+        )
+        old_value = session.execute(stmt).scalar_one_or_none()
+        if old_value != str(value):
+            old_option = models.ConfigHistory(section=section,
+                                              opt=option,
+                                              value=old_value)
+            old_option.save(session=session)
+            stmt = update(
+                models.Config
+            ).where(
+                and_(models.Config.section == section,
+                     models.Config.opt == option)
+            ).values({
+                models.Config.value: str(value)
+            })
+            session.execute(stmt)
+            delete_from_cache(key=CacheKey.value(section, option))
+            delete_from_cache(key=CacheKey.items(section))
+
+
+@transactional_session
+def remove_section(section: str, *, session: "Session") -> bool:
+    """
+    Remove the specified section from the specified section.
+
+    :param section: The name of the section.
+    :param session: The database session in use.
+    :returns: True/False.
+    """
+
+    if not has_section(section=section, session=session):
+        return False
+    else:
+        stmt = select(
+            models.Config.value
+        ).where(
+            models.Config.section == section
+        )
+        for old in session.execute(stmt).all():
+            old_option = models.ConfigHistory(section=old[0],
+                                              opt=old[1],
+                                              value=old[2])
+            old_option.save(session=session)
+            delete_from_cache(key=CacheKey.has_option(old[0], old[1]))
+            delete_from_cache(key=CacheKey.value(old[0], old[1]))
+
+        stmt = delete(
+            models.Config
+        ).where(
+            models.Config.section == section
+        )
+        session.execute(stmt)
+        delete_from_cache(key=SECTIONS_CACHE_KEY)
+        delete_from_cache(key=CacheKey.items(section))
+        return True
+
+
+@transactional_session
+def remove_option(section: str, option: str, *, session: "Session") -> bool:
+    """
+    Remove the specified option from the configuration.
+
+    :param section: The name of the section.
+    :param option: The name of the option.
+    :param session: The database session in use.
+    :returns: True/False
+    """
+
+    if not has_option(section=section, option=option, session=session, use_cache=False):
+        return False
+    else:
+        stmt = select(
+            models.Config.value
+        ).where(
+            and_(models.Config.section == section,
+                 models.Config.opt == option)
+        )
+        result = session.execute(stmt).scalar_one_or_none()
+        old_option = models.ConfigHistory(section=section,
+                                          opt=option,
+                                          value=result)
+        old_option.save(session=session)
+
+        stmt = delete(
+            models.Config
+        ).where(
+            and_(models.Config.section == section,
+                 models.Config.opt == option)
+        )
+        session.execute(stmt)
+
+        stmt = select(
+            func.count()
+        ).select_from(
+            models.Config
+        ).where(
+            models.Config.section == section
+        )
+        if not session.execute(stmt).scalar_one_or_none():
+            # we deleted the last config entry in the section. Invalidate the section cache
+            delete_from_cache(key=SECTIONS_CACHE_KEY)
+            delete_from_cache(key=CacheKey.has_section(section))
+        delete_from_cache(key=CacheKey.items(section))
+        delete_from_cache(key=CacheKey.has_option(section, option))
+        delete_from_cache(key=CacheKey.value(section, option))
+        return True
+
+
+def read_from_cache(key: str, expiration_time: int = 900) -> Any:
+    """
+    Try to read a value from a cache.
+
+    :param key: Key that stores the value.
+    :param expiration_time: Time in seconds that a value should not be older than.
+    """
+    key = key.replace(' ', '')
+    value = REGION.get(key, expiration_time=expiration_time)
+    return value
+
+
+def write_to_cache(key: str, value: Any) -> None:
+    """
+    Set a value on a key in a cache.
+
+    :param key: Key that stores the value.
+    :param value: Value to be stored.
+    """
+    key = key.replace(' ', '')
+    REGION.set(key, value)
+
+
+def delete_from_cache(key: str) -> None:
+    """
+    Delete from cache any data stored for the given key
+
+    :param key: Key that stores the value.
+    """
+    key = key.replace(' ', '')
+    REGION.delete(key)

rucio/core/credential.py

@@ -0,0 +1,224 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import datetime
+import hmac
+import time
+from hashlib import sha1
+from typing import Optional
+from urllib.parse import urlencode, urlparse
+
+import boto3
+from botocore.client import Config
+from dogpile.cache.api import NO_VALUE
+from google.oauth2.service_account import Credentials
+
+from rucio.common.cache import MemcacheRegion
+from rucio.common.config import config_get, get_rse_credentials
+from rucio.common.constants import RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS, RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, SUPPORTED_SIGN_URL_SERVICES, SUPPORTED_SIGN_URL_SERVICES_LITERAL, RseAttr
+from rucio.common.exception import UnsupportedOperation
+from rucio.core.monitor import MetricManager
+from rucio.core.rse import get_rse_attribute
+
+CREDS_GCS = None
+
+REGION = MemcacheRegion(expiration_time=900)
+METRICS = MetricManager(module=__name__)
+
+
+def get_signed_url(
+        rse_id: str,
+        service: SUPPORTED_SIGN_URL_SERVICES_LITERAL,
+        operation: RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL,
+        url: str,
+        lifetime: Optional[int] = 600
+) -> str:
+    """
+    Get a signed URL for a particular service and operation.
+
+    The signed URL will be valid for 1 hour but can be overridden.
+
+    :param rse_id: The ID of the RSE that the URL points to.
+    :param service: The service to authorise, either 'gcs', 's3' or 'swift'.
+    :param operation: The operation to sign, either 'read', 'write', or 'delete'.
+    :param url: The URL to sign.
+    :param lifetime: Lifetime of the signed URL in seconds.
+    :returns: Signed URL as a variable-length string.
+    """
+
+    global CREDS_GCS
+
+    if service not in SUPPORTED_SIGN_URL_SERVICES:
+        raise UnsupportedOperation('Service must be "gcs", "s3" or "swift"')
+
+    if operation not in RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS:
+        raise UnsupportedOperation('Operation must be "read", "write", or "delete"')
+
+    if url is None or url == '':
+        raise UnsupportedOperation('URL must not be empty')
+
+    if lifetime:
+        if not isinstance(lifetime, int):
+            try:
+                lifetime = int(lifetime)
+            except:
+                raise UnsupportedOperation('Lifetime must be convertible to numeric.')
+
+    if service == 'gcs':
+        if not CREDS_GCS:
+            CREDS_GCS = Credentials.from_service_account_file(config_get('credentials', 'gcs',
+                                                                         raise_exception=False,
+                                                                         default='/opt/rucio/etc/google-cloud-storage-test.json'))
+        components = urlparse(url)
+        host = components.netloc
+
+        # special case to test signature, force epoch time
+        if lifetime is None:
+            lifetime = 0
+        else:
+            # GCS is timezone-sensitive, don't use UTC
+            # has to be converted to Unixtime
+            lifetime_datetime = datetime.datetime.now() + datetime.timedelta(seconds=lifetime)
+            lifetime = int(time.mktime(lifetime_datetime.timetuple()))
+
+        # sign the path only
+        path = components.path
+
+        # Map operations
+        operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
+
+        # assemble message to sign
+        to_sign = "%s\n\n\n%s\n%s" % (operations[operation], lifetime, path)
+
+        # create URL-capable signature
+        # first character is always a '=', remove it
+        signature = urlencode({'': base64.b64encode(CREDS_GCS.sign_bytes(to_sign))})[1:]
+
+        # assemble final signed URL
+        signed_url = (
+            f'https://{host}{path}'
+            f'?GoogleAccessId={CREDS_GCS.service_account_email}'
+            f'&Expires={lifetime}&Signature={signature}'
+        )
+
+    elif service == 's3':
+
+        # get RSE S3 URL style (path or host)
+        # path-style: https://s3.region-code.amazonaws.com/bucket-name/key-name
+        # host-style: https://bucket-name.s3.region-code.amazonaws.com/key-name
+        s3_url_style = get_rse_attribute(rse_id, RseAttr.S3_URL_STYLE)
+
+        # no S3 URL style specified, assume path-style
+        if s3_url_style is None:
+            s3_url_style = "path"
+
+        # split URL to get hostname, bucket and key
+        components = urlparse(url)
+        host = components.netloc
+        pathcomponents = components.path.split('/')
+        if s3_url_style == "path":
+            if len(pathcomponents) < 3:
+                raise UnsupportedOperation('Not a valid Path-Style S3 URL')
+            bucket = pathcomponents[1]
+            key = '/'.join(pathcomponents[2:])
+        elif s3_url_style == "host":
+            hostcomponents = host.split('.')
+            bucket = hostcomponents[0]
+            if len(pathcomponents) < 2:
+                raise UnsupportedOperation('Not a valid Host-Style S3 URL')
+            key = '/'.join(pathcomponents[1:])
+        else:
+            raise UnsupportedOperation('Not a valid RSE S3 URL style (allowed values: path|host)')
+
+        # remove port number from host if present
+        colon = host.find(':')
+        port = '443'
+        if colon >= 0:
+            port = host[colon + 1:]
+            host = host[:colon]
+
+        # look up in RSE account configuration by RSE ID
+        cred_name = rse_id
+        cred = REGION.get('s3-%s' % cred_name)
+        if cred is NO_VALUE:
+            rse_cred = get_rse_credentials()
+            cred = rse_cred.get(cred_name)
+            REGION.set('s3-%s' % cred_name, cred)
+        access_key = cred['access_key']
+        secret_key = cred['secret_key']
+        signature_version = cred['signature_version']
+        region_name = cred['region']
+
+        if operation == 'read':
+            s3op = 'get_object'
+        elif operation == 'write':
+            s3op = 'put_object'
+        else:
+            s3op = 'delete_object'
+
+        with METRICS.timer('signs3'):
+
+            if s3_url_style == "host":
+                s3_url_style = "virtual"
+
+            s3 = boto3.client(service_name='s3',
+                              endpoint_url=f'https://{host}:{port}',
+                              aws_access_key_id=access_key,
+                              aws_secret_access_key=secret_key,
+                              config=Config(signature_version=signature_version,
+                                            region_name=region_name,
+                                            s3={"addressing_style": s3_url_style}))
+
+            signed_url: str = s3.generate_presigned_url(
+                s3op, Params={'Bucket': bucket, 'Key': key}, ExpiresIn=lifetime)
+
+    else:  # service == 'swift'
+        # split URL to get hostname and path
+        components = urlparse(url)
+        host = components.netloc
+
+        # remove port number from host if present
+        colon = host.find(':')
+        if colon >= 0:
+            host = host[:colon]
+
+        # use RSE ID to look up key
+        cred_name = rse_id
+
+        # look up tempurl signing key
+        cred = REGION.get('swift-%s' % cred_name)
+        if cred is NO_VALUE:
+            rse_cred = get_rse_credentials()
+            cred = rse_cred.get(cred_name)
+            REGION.set('swift-%s' % cred_name, cred)
+        tempurl_key = cred['tempurl_key']
+
+        if operation == 'read':
+            swiftop = 'GET'
+        elif operation == 'write':
+            swiftop = 'PUT'
+        else:
+            swiftop = 'DELETE'
+
+        expires = int(time.time() + lifetime)  # type: ignore (lifetime could be None)
+
+        # create signed URL
+        with METRICS.timer('signswift'):
+            hmac_body = '%s\n%s\n%s' % (swiftop, expires, components.path)
+            # Python 3 hmac only accepts bytes or bytearray
+            sig = hmac.new(bytearray(tempurl_key, 'utf-8'), bytearray(hmac_body, 'utf-8'), sha1).hexdigest()
+            signed_url = f'https://{host}{components.path}?temp_url_sig={sig}&temp_url_expires={expires}'
+
+    return signed_url