rucio 37.0.0rc1__py3-none-any.whl

rucio/transfertool/transfertool.py

@@ -0,0 +1,221 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from abc import ABCMeta, abstractmethod
+from typing import TYPE_CHECKING, Any, Optional
+
+from rucio.common.constants import SUPPORTED_PROTOCOLS
+from rucio.core.request import get_request
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping, Sequence
+
+    from sqlalchemy.orm import Session
+
+    from rucio.common.types import LoggerFunction
+    from rucio.core.request import DirectTransfer
+    from rucio.core.rse import RseData
+
+
+class TransferToolBuilder:
+    """
+    Builder for Transfertool objects.
+    Stores the parameters needed to create the Transfertool object of the given type/class.
+
+    Implements __hash__ and __eq__ to allow using it as key in dictionaries and group transfers
+    by common transfertool.
+    """
+    def __init__(self, transfertool_class: type["Transfertool"], **kwargs):
+        self.transfertool_class = transfertool_class
+        self.fixed_kwargs = frozenset(kwargs.items())
+
+    def __str__(self):
+        return self.transfertool_class.__name__
+
+    def __hash__(self):
+        return hash(frozenset(self.__dict__.items()))
+
+    def __eq__(self, other):
+        return self.__class__ == other.__class__ and self.__dict__ == other.__dict__
+
+    def make_transfertool(self, **additional_kwargs) -> "Transfertool":
+        all_kwargs = dict(self.fixed_kwargs)
+        all_kwargs.update(additional_kwargs)
+        return self.transfertool_class(**all_kwargs)
+
+
+class TransferStatusReport(metaclass=ABCMeta):
+    """
+    Allows to compute the changes which have to be applied to the database
+    to reflect the current status reported by the external transfertool into
+    the database.
+    """
+
+    supported_db_fields = [
+        'state',
+    ]
+
+    def __init__(self, request_id: str, request: Optional[dict[str, Any]] = None):
+        self.request_id = request_id
+        self.__request = request  # Optional: DB request. If provided, saves us a database call to fetch it by request_id
+        self.__initialized = False
+
+        # Supported db fields below
+        self.state = None
+
+    @abstractmethod
+    def initialize(self, session: "Session", logger: "LoggerFunction" = logging.log) -> None:
+        """
+        Initialize all fields from self.supported_update_fields
+        """
+        pass
+
+    @abstractmethod
+    def get_monitor_msg_fields(self, session: "Session", logger: "LoggerFunction" = logging.log) -> dict[str, Any]:
+        """
+        Return the fields which will be included in the message sent to hermes.
+        """
+        pass
+
+    def ensure_initialized(self, session: "Session", logger: "LoggerFunction" = logging.log) -> None:
+        if not self.__initialized:
+            self.initialize(session, logger)
+            self.__initialized = True
+
+    def request(self, session: "Session") -> Optional[dict[str, Any]]:
+        """
+        Fetch the request by request_id if needed.
+        """
+        if not self.__request:
+            self.__request = get_request(self.request_id, session=session)
+        return self.__request
+
+    def get_db_fields_to_update(self, session: "Session", logger: "LoggerFunction" = logging.log) -> dict[str, Any]:
+        """
+        Returns the fields which have to be updated in the request
+        """
+        self.ensure_initialized(session, logger)
+
+        updates = {}
+        for field in self.supported_db_fields:
+            field_value = getattr(self, field)
+            if field_value:
+                updates[field] = field_value
+        return updates
+
+
+class Transfertool(metaclass=ABCMeta):
+    """
+    Interface definition of the Rucio transfertool
+    """
+
+    external_name = ''
+    required_rse_attrs = ()
+    supported_schemes = set(SUPPORTED_PROTOCOLS).difference(('magnet', ))
+
+    def __init__(self, external_host: str, logger: "LoggerFunction" = logging.log):
+        """
+        Initializes the transfertool
+
+        :param external_host:   The external host where the transfertool API is running
+        """
+
+        self.external_host = external_host
+        self.logger = logger
+
+    def __str__(self):
+        return self.external_host if self.external_host is not None else self.__class__.__name__
+
+    @classmethod
+    def can_perform_transfer(cls, source_rse: "RseData", dest_rse: "RseData") -> bool:
+        """
+        Return True if this transfertool is able to perform a transfer between the given source and destination rses
+        """
+        if (
+                all(source_rse.attributes.get(attribute) is not None for attribute in cls.required_rse_attrs)
+                and all(dest_rse.attributes.get(attribute) is not None for attribute in cls.required_rse_attrs)
+        ):
+            return True
+        return False
+
+    @classmethod
+    @abstractmethod
+    def submission_builder_for_path(cls, transfer_path: list["DirectTransfer"], logger: "LoggerFunction" = logging.log) -> tuple[list["DirectTransfer"], Optional["TransferToolBuilder"]]:
+        """
+        Analyze the transfer path. If this transfertool class can submit the given transfers, return
+        a TransferToolBuilder instance capable to build transfertool objects configured for this
+        particular submission.
+        :param transfer_path:  List of DirectTransfer
+        :param logger: logger instance
+        :return: a tuple: a sub-path starting at the first node from transfer_path, and a TransfertoolBuilder instance
+        capable to submit this sub-path. Returns ([], None) if submission is impossible.
+        """
+        pass
+
+    @abstractmethod
+    @abstractmethod
+    def group_into_submit_jobs(self, transfer_paths: "Iterable[list[DirectTransfer]]") -> list[dict[str, Any]]:
+        """
+        Takes an iterable over transfer paths, and create groups which can be submitted in one call to submit()
+
+        :param transfer_paths: Iterable over (potentially multihop) transfer paths.
+        :return: list of dicts of the form {"transfers": <transfer list>, "job_params": <data blob>}
+        """
+        pass
+
+    @abstractmethod
+    def submit(self, transfers: "Iterable[DirectTransfer]", job_params: dict[str, str], timeout: Optional[int] = None) -> str:
+        """
+        Submit transfers to the transfertool.
+
+        :param transfers:    List of dictionaries describing the file transfers.
+        :param job_params:   Dictionary containing key/value pairs, for all transfers.
+        :param timeout:      Timeout in seconds.
+        :returns:            Transfertool internal identifiers.
+        """
+        pass
+
+    @abstractmethod
+    def bulk_query(self, requests_by_eid: "Mapping[str, Mapping[str, Any]]", timeout: Optional[int] = None) -> dict[str, dict[str, TransferStatusReport]]:
+        """
+        Query the status of a bulk of transfers in FTS3 via JSON.
+
+        :param requests_by_eid: dictionary {external_id1: {request_id1: request1, ...}, ...} of request to be queried
+        :returns: Transfer status information as a dictionary.
+        """
+        pass
+
+    @abstractmethod
+    def cancel(self, transfer_ids: "Sequence[str]", timeout: Optional[int] = None) -> None:
+        """
+        Cancel transfers that have been submitted to the transfertool.
+
+        :param transfer_ids: Transfertool internal transfer identifiers as list of strings.
+        :param timeout:      Timeout in seconds.
+        :returns:            True if cancellation was successful.
+        """
+        pass
+
+    @abstractmethod
+    def update_priority(self, transfer_id: str, priority: int, timeout: Optional[int] = None) -> Any:
+        """
+        Update the priority of a transfer that has been submitted to the transfertool.
+
+        :param transfer_id: Transfertool internal transfer identifier as a string.
+        :param priority:    Job priority as an integer from 1 to 5.
+        :param timeout:     Timeout in seconds.
+        :returns:           True if update was successful.
+        """
+        pass

rucio/vcsversion.py

@@ -0,0 +1,11 @@
+
+'''
+This file is automatically generated; Do not edit it. :)
+'''
+VERSION_INFO = {
+    'final': True,
+    'version': '37.0.0rc1',
+    'branch_nick': 'master',
+    'revision_id': '07eb663ed1b7e8681bfdbd7db4cc91ab9ef2c27a',
+    'revno': 13606
+}

rucio/version.py

@@ -0,0 +1,45 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from rucio.vcsversion import VERSION_INFO
+
+RUCIO_VERSION = [VERSION_INFO['version'], ]
+FINAL = VERSION_INFO['final']   # This becomes true at Release Candidate time
+
+
+def canonical_version_string() -> str:
+    """ Get the canonical string """
+    return '.'.join(filter(None, RUCIO_VERSION))
+
+
+def version_string() -> str:
+    """ Get the version string """
+    return canonical_version_string()
+
+
+def vcs_version_string() -> str:
+    """ Get the VCS version string """
+    return "%s:%s" % (VERSION_INFO['branch_nick'], VERSION_INFO['revision_id'])
+
+
+def version_string_with_vcs() -> str:
+    """ Get the version string with VCS """
+
+    return "%s-%s" % (canonical_version_string(), vcs_version_string())
+
+
+def current_version() -> str:
+    """ Get the current version """
+    components = 2 if version_string().startswith("1.") else 1
+    return ".".join(version_string().split(".")[:components])

rucio/web/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/web/rest/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/web/rest/flaskapi/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/web/rest/flaskapi/authenticated_bp.py

@@ -0,0 +1,27 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from flask import Blueprint
+
+from rucio.web.rest.flaskapi.v1.common import request_auth_env
+
+
+class AuthenticatedBlueprint(Blueprint):
+    """
+    Enforce authorisation of blueprints by default
+    returns a normal blueprint otherwise
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.before_request(request_auth_env)

rucio/web/rest/flaskapi/v1/__init__.py

@@ -0,0 +1,13 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rucio/web/rest/flaskapi/v1/accountlimits.py

@@ -0,0 +1,236 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from flask import Flask, request
+
+from rucio.common.exception import AccessDenied, AccountNotFound, RSENotFound
+from rucio.gateway.account_limit import delete_global_account_limit, delete_local_account_limit, set_global_account_limit, set_local_account_limit
+from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, generate_http_error_flask, json_parameters, param_get, response_headers
+
+if TYPE_CHECKING:
+    from flask.typing import ResponseReturnValue
+
+
+class LocalAccountLimit(ErrorHandlingMethodView):
+    def post(self, account: str, rse: str) -> 'ResponseReturnValue':
+        """
+        ---
+        summary: Create or update a local account limit
+        tags:
+          - Account Limit
+        parameters:
+        - name: account
+          in: path
+          description: The account for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        - name: rse
+          in: path
+          description: The rse for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        requestBody:
+          content:
+            'application/json':
+              schema:
+                type: object
+                required:
+                - bytes
+                properties:
+                  bytes:
+                    description: The new limit in bytes.
+                    type: integer
+        responses:
+          201:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['Created']
+          401:
+            description: Invalid Auth Token
+          404:
+            description: No RSE or account found for the given id.
+        """
+        parameters = json_parameters()
+        bytes_param = param_get(parameters, 'bytes')
+        try:
+            set_local_account_limit(account=account, rse=rse, bytes_=bytes_param, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+        except AccessDenied as error:
+            return generate_http_error_flask(401, error)
+        except (RSENotFound, AccountNotFound) as error:
+            return generate_http_error_flask(404, error)
+
+        return 'Created', 201
+
+    def delete(self, account: str, rse: str) -> 'ResponseReturnValue':
+        """
+        ---
+        summary: Delete a local account limit
+        tags:
+          - Account Limit
+        parameters:
+        - name: account
+          in: path
+          description: The account for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        - name: rse
+          in: path
+          description: The rse for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        responses:
+          200:
+            description: OK
+          401:
+            description: Invalid Auth Token
+          404:
+            description: No RSE or account found for the given id.
+        """
+        try:
+            delete_local_account_limit(account=account, rse=rse, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+        except AccessDenied as error:
+            return generate_http_error_flask(401, error)
+        except (AccountNotFound, RSENotFound) as error:
+            return generate_http_error_flask(404, error)
+
+        return '', 200
+
+
+class GlobalAccountLimit(ErrorHandlingMethodView):
+    def post(self, account: str, rse_expression: str) -> 'ResponseReturnValue':
+        """
+        ---
+        summary: Create or update a global account limit
+        tags:
+          - Account Limit
+        parameters:
+        - name: account
+          in: path
+          description: The account for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        - name: rse_expression
+          in: path
+          description: The rse expression for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        requestBody:
+          content:
+            'application/json':
+              schema:
+                type: object
+                required:
+                - bytes
+                properties:
+                  bytes:
+                    description: The new limit in bytes.
+                    type: integer
+        responses:
+          201:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['Created']
+          401:
+            description: Invalid Auth Token
+          404:
+            description: No RSE or account found for the given id.
+        """
+        parameters = json_parameters()
+        bytes_param = param_get(parameters, 'bytes')
+        try:
+            set_global_account_limit(
+                account=account,
+                rse_expression=rse_expression,
+                bytes_=bytes_param,
+                issuer=request.environ.get('issuer'),
+                vo=request.environ.get('vo'),
+            )
+        except AccessDenied as error:
+            return generate_http_error_flask(401, error)
+        except (RSENotFound, AccountNotFound) as error:
+            return generate_http_error_flask(404, error)
+
+        return 'Created', 201
+
+    def delete(self, account: str, rse_expression: str) -> 'ResponseReturnValue':
+        """
+        ---
+        summary: Delete a global account limit
+        tags:
+          - Account Limit
+        parameters:
+        - name: account
+          in: path
+          description: The account for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        - name: rse_expression
+          in: path
+          description: The rse expression for the accountlimit.
+          schema:
+            type: string
+          style: simple
+        responses:
+          200:
+            description: OK
+          401:
+            description: Invalid Auth Token
+          404:
+            description: No RSE or account found for the given id.
+        """
+        try:
+            delete_global_account_limit(account=account, rse_expression=rse_expression, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+        except AccessDenied as error:
+            return generate_http_error_flask(401, error)
+        except (AccountNotFound, RSENotFound) as error:
+            return generate_http_error_flask(404, error)
+
+        return '', 200
+
+
+def blueprint(with_doc: bool = False) -> AuthenticatedBlueprint:
+    bp = AuthenticatedBlueprint('accountlimits', __name__, url_prefix='/accountlimits')
+
+    local_account_limit_view = LocalAccountLimit.as_view('local_account_limit')
+    bp.add_url_rule('/local/<account>/<rse>', view_func=local_account_limit_view, methods=['post', 'delete'])
+    if not with_doc:
+        bp.add_url_rule('/<account>/<rse>', view_func=local_account_limit_view, methods=['post', 'delete'])
+    global_account_limit_view = GlobalAccountLimit.as_view('global_account_limit')
+    bp.add_url_rule('/global/<account>/<rse_expression>', view_func=global_account_limit_view, methods=['post', 'delete'])
+
+    bp.after_request(response_headers)
+    return bp
+
+
+def make_doc() -> Flask:
+    """ Only used for sphinx documentation """
+    doc_app = Flask(__name__)
+    doc_app.register_blueprint(blueprint(with_doc=True))
+    return doc_app