rucio 37.0.0rc1__py3-none-any.whl

rucio/web/rest/flaskapi/v1/locks.py

@@ -0,0 +1,358 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Flask, request
+
+from rucio.common.exception import RSENotFound
+from rucio.common.utils import render_json
+from rucio.gateway.lock import get_dataset_locks, get_dataset_locks_bulk, get_dataset_locks_by_rse
+from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parse, parse_scope_name, response_headers, try_stream
+
+
+class LockByRSE(ErrorHandlingMethodView):
+    """ REST APIs for dataset locks. """
+
+    @check_accept_header_wrapper_flask(['application/x-json-stream'])
+    def get(self, rse):
+        """
+        ---
+        summary: Get locks by rse
+        description: Get all dataset locks for an associated rse.
+        tags:
+          - Lock
+        parameters:
+        - name: rse
+          in: path
+          description: The rse name.
+          schema:
+            type: string
+          style: simple
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  did_type:
+                    description: The did type to filter for.
+                    type: string
+                    enum: ['dataset']
+        responses:
+          200:
+            description: OK
+            content:
+              application/x-json-stream:
+                schema:
+                  description: Locks associated with the rse.
+                  type: array
+                  items:
+                    description: A lock
+                    type: object
+                    properties:
+                      rse_id:
+                        description: The id of the associated rse.
+                        type: string
+                      rse:
+                        description: The name of the associated rse.
+                        type: string
+                      scope:
+                        description: The scope of the associated rse.
+                        type: string
+                      name:
+                        description: The name of the rule.
+                        type: string
+                      rule_id:
+                        description: The id of the rule.
+                        type: string
+                      account:
+                        description: The associated account.
+                        type: string
+                      state:
+                        description: The state of the rule.
+                        type: string
+                        enum: ['R', 'O', 'S']
+                      length:
+                        description: The length of the rule.
+                        type: integer
+                      bytes:
+                        description: The bytes limit for the lock.
+                        type: integer
+                      accessed_at:
+                        description: The last time is was accessed.
+                        type: string
+          401:
+            description: Invalid Auth Token
+          500:
+            description: Wrong did type
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['wrong did_type specified']
+          406:
+            description: Not acceptable
+        """
+        did_type = request.args.get('did_type', default=None)
+        if did_type != 'dataset':
+            return 'Wrong did_type specified', 500
+
+        try:
+            def generate(vo):
+                for lock in get_dataset_locks_by_rse(rse, vo=vo):
+                    yield render_json(**lock) + '\n'
+
+            return try_stream(generate(vo=request.environ.get('vo')))
+        except RSENotFound as error:
+            return generate_http_error_flask(404, error)
+
+
+class LocksByScopeName(ErrorHandlingMethodView):
+    """ REST APIs for dataset locks. """
+
+    @check_accept_header_wrapper_flask(['application/x-json-stream'])
+    def get(self, scope_name):
+        """
+        ---
+        summary: Get locks by scope
+        description: Get all dataset locks for an associated rse.
+        tags:
+          - Lock
+        parameters:
+        - name: scope_name
+          in: path
+          description: The scope name.
+          schema:
+            type: string
+          style: simple
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  did_type:
+                    description: The did type to filter for.
+                    type: string
+                    enum: ['dataset']
+        responses:
+          200:
+            description: OK
+            content:
+              application/x-json-stream:
+                schema:
+                  description: Locks associated with the rse.
+                  type: array
+                  items:
+                    description: A lock
+                    type: object
+                    properties:
+                      rse_id:
+                        description: The id of the associated rse.
+                        type: string
+                      rse:
+                        description: The name of the associated rse.
+                        type: string
+                      scope:
+                        description: The scope of the associated rse.
+                        type: string
+                      name:
+                        description: The name of the rule.
+                        type: string
+                      rule_id:
+                        description: The id of the rule.
+                        type: string
+                      account:
+                        description: The associated account.
+                        type: string
+                      state:
+                        description: The state of the rule.
+                        type: string
+                        enum: ['R', 'O', 'S']
+                      length:
+                        description: The length of the rule.
+                        type: integer
+                      bytes:
+                        description: The bytes limit for the lock.
+                        type: integer
+                      accessed_at:
+                        description: The last time is was accessed.
+                        type: string
+          401:
+            description: Invalid Auth Token
+          500:
+            description: Wrong did type
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['wrong did_type specified']
+          406:
+            description: Not acceptable
+        """
+        did_type = request.args.get('did_type', default=None)
+        if did_type != 'dataset':
+            return 'Wrong did_type specified', 500
+
+        try:
+            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
+
+            def generate(vo):
+                for lock in get_dataset_locks(scope, name, vo=vo):
+                    yield render_json(**lock) + '\n'
+
+            return try_stream(generate(vo=request.environ.get('vo')))
+        except ValueError as error:
+            return generate_http_error_flask(400, error)
+
+
+class DatasetLocksForDids(ErrorHandlingMethodView):
+    """ REST APIs for multiple dataset locks. """
+
+    @check_accept_header_wrapper_flask(['application/x-json-stream'])
+    def post(self):
+        """ get locks for a given scope, name.
+
+        :resheader Content-Type: application/x-json-stream
+        :status 200: OK.
+        :status 400: Wrong DID type.
+        :returns: Line separated list of dictionary with lock information.
+        ---
+        summary: Get locks by dids
+        description: Get all dataset locks for the associated dids.
+        tags:
+          - Lock
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  dids:
+                    description: The dids associated with the locks.
+                    type: array
+                    items:
+                      type: object
+                      description: A did
+                      required:
+                        - scope
+                        - name
+                      properties:
+                        scope:
+                          description: The scope of the did.
+                          type: string
+                        name:
+                          description: The name of the did.
+                          type: string
+                        type:
+                          description: The type of the did.
+                          type: string
+                          enum: ['dataset', 'container']
+        responses:
+          200:
+            description: OK
+            content:
+              application/x-json-stream:
+                schema:
+                  description: Locks associated with the rse.
+                  type: array
+                  items:
+                    description: A lock
+                    type: object
+                    properties:
+                      rse_id:
+                        description: The id of the associated rse.
+                        type: string
+                      rse:
+                        description: The name of the associated rse.
+                        type: string
+                      scope:
+                        description: The scope of the associated rse.
+                        type: string
+                      name:
+                        description: The name of the rule.
+                        type: string
+                      rule_id:
+                        description: The id of the rule.
+                        type: string
+                      account:
+                        description: The associated account.
+                        type: string
+                      state:
+                        description: The state of the rule.
+                        type: string
+                        enum: ['R', 'O', 'S']
+                      length:
+                        description: The length of the rule.
+                        type: integer
+                      bytes:
+                        description: The bytes limit for the lock.
+                        type: integer
+                      accessed_at:
+                        description: The last time is was accessed.
+                        type: string
+          401:
+            description: Invalid Auth Token
+          400:
+            description: Wrong did type
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['Can not find the list of DIDs in the data. Use "dids" keyword.']
+          406:
+            description: Not acceptable
+
+        """
+
+        data = json_parse(types=(dict,))
+        try:
+            dids = data["dids"]
+        except KeyError:
+            return 'Can not find the list of DIDs in the data. Use "dids" keyword.', 400
+        vo = request.environ.get('vo')
+        try:
+            locks = get_dataset_locks_bulk(dids, vo)        # removes duplicates
+
+            def generate(locks):
+                for lock in locks:
+                    lock["scope"] = str(lock["scope"])
+                    yield render_json(**lock) + '\n'
+            return try_stream(generate(locks))
+
+        except ValueError as error:
+            return generate_http_error_flask(400, error)
+
+
+def blueprint():
+    bp = AuthenticatedBlueprint('locks', __name__, url_prefix='/locks')
+
+    lock_by_rse_view = LockByRSE.as_view('lock_by_rse')
+    bp.add_url_rule('/<rse>', view_func=lock_by_rse_view, methods=['get', ])
+
+    lock_by_scope_name_view = LocksByScopeName.as_view('locks_by_scope_name')
+    bp.add_url_rule('/<path:scope_name>', view_func=lock_by_scope_name_view, methods=['get', ])
+
+    locks_for_dids_view = DatasetLocksForDids.as_view('locks_for_dids')
+    bp.add_url_rule('/bulk_locks_for_dids', view_func=locks_for_dids_view, methods=['post', ])
+
+    bp.after_request(response_headers)
+    return bp
+
+
+def make_doc():
+    """ Only used for sphinx documentation """
+    doc_app = Flask(__name__)
+    doc_app.register_blueprint(blueprint())
+    return doc_app

rucio/web/rest/flaskapi/v1/main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import logging
+
+from flask import Flask
+
+from rucio.common.config import config_get
+from rucio.common.exception import ConfigurationError
+from rucio.common.logging import setup_logging
+from rucio.web.rest.flaskapi.v1.common import CORSMiddleware
+
+DEFAULT_ENDPOINTS = [
+    'accountlimits',
+    'accounts',
+    'auth',
+    'config',
+    'credentials',
+    'dids',
+    'dirac',
+    'export',
+    'heartbeats',
+    'identities',
+    'import',
+    'lifetime_exceptions',
+    'locks',
+    'meta_conventions',
+    'ping',
+    'redirect',
+    'replicas',
+    'requests',
+    'rses',
+    'rules',
+    'scopes',
+    'subscriptions',
+]
+
+
+def apply_endpoints(app, modules):
+    for blueprint_module in modules:
+        # Legacy patch - TODO Remove in 38.0.0
+        if blueprint_module == "meta":
+            logging.log(logging.WARNING, "Endpoint `meta` is depreciated and will be removed in future releases")
+            blueprint_module = "meta_conventions"
+        try:
+            # searches for module names locally
+            blueprint_module = importlib.import_module('.' + blueprint_module,
+                                                       package='rucio.web.rest.flaskapi.v1')
+        except ImportError:
+            raise ConfigurationError(f'Could not load "{blueprint_module}" provided in the endpoints configuration value')
+
+        if hasattr(blueprint_module, 'blueprint'):
+            app.register_blueprint(blueprint_module.blueprint())
+
+            if hasattr(blueprint_module, "blueprint_legacy"):
+                app.register_blueprint(blueprint_module.blueprint_legacy())
+
+        else:
+            raise ConfigurationError(f'"{blueprint_module}" from the endpoints configuration value did not have a blueprint')
+
+
+try:
+    endpoints = config_get('api', 'endpoints', raise_exception=False, default='')
+    endpoints = list(filter(bool, map(str.strip, endpoints.split(sep=','))))
+except RuntimeError:
+    endpoints = None
+
+if not endpoints:
+    endpoints = DEFAULT_ENDPOINTS
+
+application = Flask(__name__)
+application.wsgi_app = CORSMiddleware(application.wsgi_app)
+apply_endpoints(application, endpoints)
+setup_logging(application)
+
+
+if __name__ == '__main__':
+    application.run()

rucio/web/rest/flaskapi/v1/meta_conventions.py

@@ -0,0 +1,241 @@
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Flask, jsonify, request
+
+from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedKeyType, UnsupportedValueType
+from rucio.gateway.meta_conventions import add_key, add_value, list_keys, list_values
+from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers
+
+
+class MetaConventions(ErrorHandlingMethodView):
+    """ REST APIs for managing data identifier attribute metadata key formats. """
+
+    @check_accept_header_wrapper_flask(['application/json'])
+    def get(self):
+        """
+        ---
+        summary: List all data identifier keys.
+        tags:
+            - Meta
+        responses:
+          200:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: array
+                  description: List of all DID keys.
+                  items:
+                    type: string
+                    description: Data Itentifier key
+          401:
+            description: Invalid Auth Token
+          406:
+            description: Not acceptable
+        """
+        return jsonify(list_keys())
+
+    def post(self, key):
+        """
+        ---
+        summary: Create key
+        description: Creates a new allowed key (value is NULL).
+        tags:
+            - Meta
+        parameters:
+        - name: key
+          in: path
+          description: The name of the key.
+          schema:
+            type: string
+          style: simple
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  key_type:
+                    description: The key type.
+                    type: string
+                  value_type:
+                    description: The value type.
+                    type: string
+                  value_regexp:
+                    description: The value regexpression.
+                    type: string
+        responses:
+          201:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['Created']
+          400:
+            description: Cannot decode json parameter list / Unsupported value type.
+          401:
+            description: Invalid Auth Token.
+          409:
+            description: Key already exists.
+        """
+        parameters = json_parameters()
+
+        try:
+            add_key(
+                key=key,
+                key_type=param_get(parameters, 'key_type', default=None),
+                value_type=param_get(parameters, 'value_type', default=None),
+                value_regexp=param_get(parameters, 'value_regexp', default=None),
+                issuer=request.environ.get('issuer'),
+                vo=request.environ.get('vo'),
+            )
+        except Duplicate as error:
+            return generate_http_error_flask(409, error)
+        except (UnsupportedValueType, UnsupportedKeyType) as error:
+            return generate_http_error_flask(400, error)
+
+        return 'Created', 201
+
+
+class Values(ErrorHandlingMethodView):
+    """ REST APIs for data identifier attribute values. """
+
+    @check_accept_header_wrapper_flask(['application/json'])
+    def get(self, key):
+        """
+        ---
+        summary: Get value for key
+        description: List all values for a key.
+        tags:
+            - Meta
+        parameters:
+        - name: key
+          in: path
+          description: The reference key.
+          schema:
+            type: string
+          style: simple
+        responses:
+          200:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  description: List of all key values.
+                  type: array
+                  items:
+                    type: string
+                    description: A value associated with a key.
+          401:
+            description: Invalid Auth Token
+          406:
+            description: Not acceptable
+        """
+        return jsonify(list_values(key=key))
+
+    def post(self, key):
+        """
+        ---
+        summary: Create value for key
+        description: Creates a new value for a key.
+        tags:
+            - Meta
+        parameters:
+        - name: key
+          in: path
+          description: The reference key.
+          schema:
+            type: string
+          style: simple
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                - value
+                properties:
+                  value:
+                    description: The new value associated with a key.
+                    type: string
+        responses:
+          201:
+            description: OK
+            content:
+              application/json:
+                schema:
+                  type: string
+                  enum: ['Created']
+          400:
+            description: Cannot decode json parameter list / Invalid value for key.
+          401:
+            description: Invalid Auth Token
+          404:
+            description: Key not found
+          409:
+            description: Value already exists.
+        """
+        parameters = json_parameters()
+        value = param_get(parameters, 'value')
+
+        try:
+            add_value(key=key, value=value, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+        except Duplicate as error:
+            return generate_http_error_flask(409, error)
+        except InvalidValueForKey as error:
+            return generate_http_error_flask(400, error)
+        except KeyNotFound as error:
+            return generate_http_error_flask(404, error)
+
+        return 'Created', 201
+
+
+def blueprint():
+    bp = AuthenticatedBlueprint('meta_conventions', __name__, url_prefix='/meta_conventions')
+
+    meta_view = MetaConventions.as_view('meta_conventions')
+    bp.add_url_rule('/', view_func=meta_view, methods=['get', ])
+    bp.add_url_rule('/<key>', view_func=meta_view, methods=['post', ])
+    values_view = Values.as_view('values')
+    bp.add_url_rule('/<key>/', view_func=values_view, methods=['get', 'post'])
+
+    bp.after_request(response_headers)
+    return bp
+
+
+def blueprint_legacy():
+    # TODO: Remove in 38.0
+    bp = AuthenticatedBlueprint('meta', __name__, url_prefix='/meta')
+
+    meta_view = MetaConventions.as_view('meta')
+    bp.add_url_rule('/', view_func=meta_view, methods=['get', ])
+    bp.add_url_rule('/<key>', view_func=meta_view, methods=['post', ])
+    values_view = Values.as_view('values')
+    bp.add_url_rule('/<key>/', view_func=values_view, methods=['get', 'post'])
+
+    bp.after_request(response_headers)
+    return bp
+
+
+def make_doc():
+    """ Only used for sphinx documentation """
+    doc_app = Flask(__name__)
+
+    doc_app.register_blueprint(blueprint())
+    doc_app.register_blueprint(blueprint_legacy())
+
+    return doc_app