rucio 37.0.0rc1__py3-none-any.whl

rucio/web/rest/flaskapi/v1/metrics.py

@@ -0,0 +1,36 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Blueprint, Flask
+
+from rucio.core.monitor import generate_prometheus_metrics
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView
+
+
+class Metrics(ErrorHandlingMethodView):
+    def get(self):
+        return generate_prometheus_metrics()
+
+
+def blueprint(standalone=False):
+    bp = Blueprint('metrics', __name__, url_prefix='/' if standalone else '/metrics')
+    metrics_view = Metrics.as_view('metrics')
+    bp.add_url_rule('/', view_func=metrics_view, methods=['get', ])
+    return bp
+
+
+def make_doc():
+    doc_app = Flask(__name__)
+    doc_app.register_blueprint(blueprint())
+    return doc_app

rucio/web/rest/flaskapi/v1/nongrid_traces.py

@@ -0,0 +1,97 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from typing import TYPE_CHECKING
+
+from flask import Blueprint, Flask, request
+from werkzeug.datastructures import Headers
+
+from rucio.core.nongrid_trace import trace
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, json_parameters, response_headers
+
+if TYPE_CHECKING:
+    from typing import Optional
+
+    from rucio.web.rest.flaskapi.v1.types import HeadersType
+
+
+class XAODTrace(ErrorHandlingMethodView):
+
+    def get_headers(self) -> "Optional[HeadersType]":
+        headers = Headers()
+        headers.set('Content-Type', 'application/octet-stream')
+        headers.set('Access-Control-Allow-Origin', request.environ.get('HTTP_ORIGIN'))
+        headers.set('Access-Control-Allow-Headers', request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
+        headers.set('Access-Control-Allow-Methods', '*')
+        headers.set('Access-Control-Allow-Credentials', 'true')
+        return headers
+
+    def post(self):
+        """
+        ---
+        summary: Trace endpoints
+        description: Trace endpoint used by the XAOD framework to post data access information.
+        tags:
+          - Nongrid traces
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  payload:
+                    description: Dictionary containing the trace information.
+                    type: object
+        responses:
+          201:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ["Created"]
+          400:
+            description: Cannot decode json data.
+        """
+        headers = self.get_headers()
+
+        parameters = json_parameters()
+
+        # generate entry timestamp
+        parameters['timeentry'] = int(time.time())
+
+        # guess client IP
+        parameters['ip'] = request.headers.get('X-Forwarded-For', default=request.remote_addr)
+
+        trace(payload=parameters)
+
+        return 'Created', 201, headers
+
+
+def blueprint():
+    bp = Blueprint('nongrid_traces', __name__, url_prefix='/nongrid_traces')
+
+    xaod_trace_view = XAODTrace.as_view('xaod_trace')
+    bp.add_url_rule('/', view_func=xaod_trace_view, methods=['post', ])
+
+    bp.after_request(response_headers)
+    return bp
+
+
+def make_doc():
+    """ Only used for sphinx documentation """
+    doc_app = Flask(__name__)
+    doc_app.register_blueprint(blueprint())
+    return doc_app

rucio/web/rest/flaskapi/v1/ping.py

@@ -0,0 +1,88 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from flask import Blueprint, Flask, jsonify, request
+from werkzeug.datastructures import Headers
+
+from rucio import version
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers
+
+if TYPE_CHECKING:
+    from typing import Optional
+
+    from rucio.web.rest.flaskapi.v1.types import HeadersType
+
+
+class Ping(ErrorHandlingMethodView):
+
+    def get_headers(self) -> "Optional[HeadersType]":
+        headers = Headers()
+        headers.set('Access-Control-Allow-Origin', request.environ.get('HTTP_ORIGIN'))
+        headers.set('Access-Control-Allow-Headers', request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
+        headers.set('Access-Control-Allow-Methods', '*')
+        headers.set('Access-Control-Allow-Credentials', 'true')
+
+        headers.set('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate')
+        headers.add('Cache-Control', 'post-check=0, pre-check=0')
+        headers.set('Pragma', 'no-cache')
+        return headers
+
+    @check_accept_header_wrapper_flask(['application/json'])
+    def get(self):
+        """
+        ---
+        summary: Ping
+        description: Ping the server and get data about it.
+        tags:
+          - Ping
+        responses:
+          200:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: object
+                  properties:
+                    version:
+                      description: The server version.
+                      type: string
+          406:
+            description: Not acceptable
+        """
+        headers = self.get_headers()
+        response = jsonify(version=version.version_string())
+        response.headers.extend(headers)
+        return response
+
+
+def blueprint(standalone=False, with_doc=False):
+    bp = Blueprint('ping', __name__, url_prefix='/' if standalone else '/ping')
+
+    ping_view = Ping.as_view('ping')
+    if not with_doc:
+        # rule without trailing slash needs to be added before rule with trailing slash
+        bp.add_url_rule('', view_func=ping_view, methods=['get', ])
+    bp.add_url_rule('/', view_func=ping_view, methods=['get', ])
+
+    bp.after_request(response_headers)
+    return bp
+
+
+def make_doc():
+    """ Only used for sphinx documentation to add the prefix """
+    doc_app = Flask(__name__)
+    doc_app.register_blueprint(blueprint(with_doc=True))
+    return doc_app

rucio/web/rest/flaskapi/v1/redirect.py

@@ -0,0 +1,366 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+from typing import TYPE_CHECKING
+
+from flask import Blueprint, Flask, redirect, request
+from werkzeug.datastructures import Headers
+
+from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound, SortingAlgorithmNotSupported
+from rucio.core.replica_sorter import site_selector, sort_replicas
+from rucio.gateway.replica import list_replicas
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, extract_vo, generate_http_error_flask, parse_scope_name, try_stream
+
+if TYPE_CHECKING:
+    from typing import Optional
+
+    from rucio.common.types import IPDict, ReplicaDict
+    from rucio.web.rest.flaskapi.v1.types import HeadersType
+
+
+class MetaLinkRedirector(ErrorHandlingMethodView):
+
+    def get_headers(self) -> "Optional[HeadersType]":
+        headers = Headers()
+        headers.set('Access-Control-Allow-Origin', request.environ.get('HTTP_ORIGIN'))
+        headers.set('Access-Control-Allow-Headers', request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
+        headers.set('Access-Control-Allow-Methods', '*')
+        headers.set('Access-Control-Allow-Credentials', 'true')
+        return headers
+
+    @check_accept_header_wrapper_flask(['application/metalink4+xml'])
+    def get(self, scope_name):
+        """
+        ---
+        summary: Metalink redirect
+        description: Get Metalink redirect.
+        tags:
+          - Redirect
+        parameters:
+        - name: scope_name
+          in: path
+          description: The data identifier (scope)/(name).
+          schema:
+            type: string
+          style: simple
+        - name: ip
+          in: query
+          description: The client ip.
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: fqdn
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: site
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: schemes
+          in: query
+          schema:
+            type: array
+          style: simple
+          required: false
+        - name: select
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: sort
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        responses:
+          200:
+            description: OK
+            content:
+              application/metalink4+xml:
+                schema:
+                  description: The metalink file.
+                  type: string
+          401:
+            description: Invalid Auth Token
+          404:
+            description: Rse or did not found
+          406:
+            description: Not acceptable
+        """
+        headers = self.get_headers()
+
+        try:
+            scope, name = parse_scope_name(scope_name, extract_vo(request.headers))
+        except ValueError as error:
+            return generate_http_error_flask(400, error, headers=headers)
+
+        # set the correct client IP
+        client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
+
+        client_location: 'IPDict' = {
+            'ip': request.args.get('ip', default=client_ip),
+            'fqdn': request.args.get('fqdn', default=None),
+            'site': request.args.get('site', default=None),
+        }
+
+        dids = [{'scope': scope, 'name': name}]
+        schemes = request.args.getlist('schemes') or ['http', 'https', 'root', 'gsiftp', 'srm', 'davs']
+        sortby = request.args.get('select', default=None)
+        sortby = request.args.get('sort', default=sortby)
+
+        # get vo if given
+        vo = extract_vo(request.headers)
+
+        try:
+            replicas_iter = list_replicas(dids=dids, schemes=schemes, client_location=client_location, vo=vo)
+            try:
+                first = next(replicas_iter)
+            except StopIteration:
+                return 'no redirection possible - cannot find the DID', 404
+
+            def generate():
+                # first, set the appropriate content type, and stream the header
+                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
+
+                # iteratively stream the XML per file
+                for rfile in itertools.chain((first,), replicas_iter):
+                    replicas = []
+                    dictreplica = {}
+                    for rse in rfile['rses']:
+                        for replica in rfile['rses'][rse]:
+                            replicas.append(replica)
+                            dictreplica[replica] = rse
+
+                    # stream metadata
+                    yield ' <file name="' + rfile['name'] + '">\n'
+                    yield '  <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
+
+                    if rfile['adler32'] is not None:
+                        yield '  <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
+                    if rfile['md5'] is not None:
+                        yield '  <hash type="md5">' + rfile['md5'] + '</hash>\n'
+
+                    yield '  <size>' + str(rfile['bytes']) + '</size>\n'
+
+                    yield f'  <glfn name="/atlas/rucio/{rfile["scope"]}:{rfile["name"]}">'
+                    yield '</glfn>\n'
+
+                    replicas = sort_replicas(dictreplica, client_location, selection=sortby)
+
+                    # stream URLs
+                    idx = 1
+                    for replica in replicas:
+                        yield '  <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx) + '">' + replica + '</url>\n'
+                        idx += 1
+
+                    yield ' </file>\n'
+
+                # don't forget to send the metalink footer
+                yield '</metalink>\n'
+
+            return try_stream(generate(), content_type='application/metalink4+xml')
+        except (DataIdentifierNotFound, ReplicaNotFound, SortingAlgorithmNotSupported) as error:
+            return generate_http_error_flask(404, error, headers=headers)
+
+
+class HeaderRedirector(ErrorHandlingMethodView):
+
+    def get_headers(self) -> "Optional[HeadersType]":
+        headers = Headers()
+        headers.set('Access-Control-Allow-Origin', request.environ.get('HTTP_ORIGIN'))
+        headers.set('Access-Control-Allow-Headers', request.environ.get('HTTP_ACCESS_CONTROL_REQUEST_HEADERS'))
+        headers.set('Access-Control-Allow-Methods', '*')
+        headers.set('Access-Control-Allow-Credentials', 'true')
+        return headers
+
+    def get(self, scope_name):
+        """
+        ---
+        summary: Header redirect
+        description: Get the header redirect.
+        tags:
+          - Redirect
+        parameters:
+        - name: scope_name
+          in: path
+          description: The data identifier (scope)/(name).
+          schema:
+            type: string
+          style: simple
+        - name: ip
+          in: query
+          description: The client ip.
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: fqdn
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: site
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: schemes
+          in: query
+          schema:
+            type: array
+          style: simple
+          required: false
+        - name: select
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: sort
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        - name: rse
+          in: query
+          schema:
+            type: string
+          style: simple
+          required: false
+        responses:
+          200:
+            description: OK
+          303:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  description: The redirect url.
+                  type: string
+          401:
+            description: Invalid Auth Token
+          404:
+            description: Rse or did not found
+        """
+        headers = self.get_headers()
+
+        try:
+            scope, name = parse_scope_name(scope_name, extract_vo(request.headers))
+        except ValueError as error:
+            return generate_http_error_flask(400, error, headers=headers)
+
+        try:
+            client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
+
+            client_location: 'IPDict' = {
+                'ip': request.args.get('ip', default=client_ip),
+                'fqdn': request.args.get('fqdn', default=None),
+                'site': request.args.get('site', default=None),
+            }
+            # use the default HTTP protocols if no scheme is given
+            schemes = request.args.getlist('schemes') or ['davs', 'https', 's3']
+            sortby = request.args.get('select', default='random')
+            sortby = request.args.get('sort', default=sortby)
+            rse = request.args.get('rse', default=None)
+            site = request.args.get('site', default=None)
+
+            # correctly forward the schemes and select to potential metalink followups
+            cleaned_url = request.environ.get('REQUEST_URI').split('?')[0]
+
+            headers.set('Link', f'<{cleaned_url}/metalink?schemes={",".join(schemes)}&select={sortby}>; rel=describedby; type="application/metalink+xml"')
+
+            # get vo if given
+            vo = extract_vo(request.headers)
+
+            replicas = list(
+                list_replicas(  # type: ignore (session parameter missing)
+                    dids=[{'scope': scope, 'name': name, 'type': 'FILE'}],
+                    schemes=schemes,
+                    client_location=client_location,
+                    vo=vo
+                )
+            )
+
+            selected_url = None
+            for r in replicas:
+                if r['rses']:
+                    dictreplica: dict[str, 'ReplicaDict'] = {}
+
+                    if rse:
+                        if rse in r['rses'] and r['rses'][rse]:
+                            selected_url = r['rses'][rse][0]
+                        else:
+                            return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
+                    else:
+
+                        for rep in r['rses']:
+                            for replica in r['rses'][rep]:
+                                # since this is HTTP-only redirection, and to ensure compatibility with as many http clients as possible
+                                # forcibly replacement davs and s3 URLs to https
+                                replica = replica.replace('davs://', 'https://').replace('s3://', 'https://')
+                                dictreplica[replica] = rep
+
+                        if not dictreplica:
+                            return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
+
+                        elif site:
+                            rep = site_selector(dictreplica, site, vo)
+                            if rep:
+                                selected_url = rep[0]
+                            else:
+                                return 'no redirection possible - no valid RSE for HTTP redirection found', 404, headers
+                        else:
+                            rep = sort_replicas(dictreplica, client_location, selection=sortby)
+                            selected_url = rep[0]
+
+            if selected_url:
+                response = redirect(selected_url, code=303)
+                response.headers.extend(headers)
+                return response
+
+            return 'no redirection possible - file does not exist', 404, headers
+        except (ReplicaNotFound, SortingAlgorithmNotSupported) as error:
+            return generate_http_error_flask(404, error, headers=headers)
+
+
+def blueprint(with_doc=False):
+    bp = Blueprint('redirect', __name__, url_prefix='/redirect')
+
+    metalink_redirector_view = MetaLinkRedirector.as_view('metalink_redirector')
+    bp.add_url_rule('/<path:scope_name>/metalink', view_func=metalink_redirector_view, methods=['get', ])
+    header_redirector_view = HeaderRedirector.as_view('header_redirector')
+    bp.add_url_rule('/<path:scope_name>', view_func=header_redirector_view, methods=['get', ])
+    if not with_doc:
+        bp.add_url_rule('/<path:scope_name>/', view_func=header_redirector_view, methods=['get', ])
+
+    return bp
+
+
+def make_doc():
+    """ Only used for sphinx documentation """
+    doc_app = Flask(__name__)
+    doc_app.register_blueprint(blueprint(with_doc=True))
+    return doc_app