rucio 37.2.0__py3-none-any.whl → 37.4.0__py3-none-any.whl

rucio/web/rest/flaskapi/v1/replicas.py

@@ -226,7 +226,7 @@ class Replicas(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
+            scope, name = parse_scope_name(scope_name, request.environ['vo'])
         except ValueError as error:
             return generate_http_error_flask(400, error)
 
@@ -252,7 +252,7 @@ class Replicas(ErrorHandlingMethodView):
             def _list_and_sort_replicas(vo):
                 # we need to call list_replicas before starting to reply
                 # otherwise the exceptions won't be propagated correctly
-                for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
+                for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):  # type: ignore (pending https://github.com/rucio/rucio/issues/7739)
                     replicas = []
                     dictreplica = {}
                     for rse in rfile['rses']:
@@ -264,7 +264,7 @@ class Replicas(ErrorHandlingMethodView):
                     rfile['pfns'] = dict(_sorted_with_priorities(rfile['pfns'], replicas, limit=limit))
                     yield rfile
 
-            rfiles = _list_and_sort_replicas(vo=request.environ.get('vo'))
+            rfiles = _list_and_sort_replicas(vo=request.environ['vo'])
             if metalink:
                 response_generator = _generate_metalink_response(rfiles, 'atlas', detailed_url=False)
             else:
@@ -359,8 +359,8 @@ class Replicas(ErrorHandlingMethodView):
             add_replicas(
                 rse=rse,
                 files=files,
-                issuer=request.environ.get('issuer'),
-                vo=request.environ.get('vo'),
+                issuer=request.environ['issuer'],
+                vo=request.environ['vo'],
                 ignore_availability=param_get(parameters, 'ignore_availability', default=False),
             )
         except InvalidPath as error:
@@ -432,7 +432,7 @@ class Replicas(ErrorHandlingMethodView):
         files = param_get(parameters, 'files')
 
         try:
-            update_replicas_states(rse=rse, files=files, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            update_replicas_states(rse=rse, files=files, issuer=request.environ['issuer'], vo=request.environ['vo'])
         except AccessDenied as error:
             return generate_http_error_flask(401, error)
 
@@ -486,8 +486,8 @@ class Replicas(ErrorHandlingMethodView):
             delete_replicas(
                 rse=rse,
                 files=files,
-                issuer=request.environ.get('issuer'),
-                vo=request.environ.get('vo'),
+                issuer=request.environ['issuer'],
+                vo=request.environ['vo'],
                 ignore_availability=param_get(parameters, 'ignore_availability', default=False),
             )
         except AccessDenied as error:
@@ -746,8 +746,8 @@ class ListReplicas(ErrorHandlingMethodView):
                     yield rfile
 
             rfiles = _list_and_sort_replicas(request_id=request.environ.get('request_id'),
-                                             issuer=request.environ.get('issuer'),
-                                             vo=request.environ.get('vo'))
+                                             issuer=request.environ['issuer'],
+                                             vo=request.environ['vo'])
             if metalink:
                 policy_schema = config_get('policy', 'schema', raise_exception=False, default='generic')
                 response_generator = _generate_metalink_response(rfiles, policy_schema)
@@ -822,7 +822,7 @@ class ReplicasDIDs(ErrorHandlingMethodView):
                 for pfn in get_did_from_pfns(pfns, rse, vo=vo):
                     yield dumps(pfn) + '\n'
 
-            return try_stream(generate(vo=request.environ.get('vo')))
+            return try_stream(generate(vo=request.environ['vo']))
         except AccessDenied as error:
             return generate_http_error_flask(401, error)
 
@@ -882,7 +882,7 @@ class BadReplicas(ErrorHandlingMethodView):
 
         try:
             not_declared_files = declare_bad_file_replicas(replicas, reason=reason,
-                                                           issuer=request.environ.get('issuer'), vo=request.environ.get('vo'),
+                                                           issuer=request.environ['issuer'], vo=request.environ['vo'],
                                                            force=force)
             return not_declared_files, 201
         except AccessDenied as error:
@@ -1002,7 +1002,7 @@ class SuspiciousReplicas(ErrorHandlingMethodView):
         reason = param_get(parameters, 'reason', default=None)
 
         try:
-            not_declared_files = declare_suspicious_file_replicas(pfns=pfns, reason=reason, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            not_declared_files = declare_suspicious_file_replicas(pfns=pfns, reason=reason, issuer=request.environ['issuer'], vo=request.environ['vo'])
             return not_declared_files, 201
         except AccessDenied as error:
             return generate_http_error_flask(401, error)
@@ -1081,7 +1081,7 @@ class SuspiciousReplicas(ErrorHandlingMethodView):
             if 'nattempts' in params:
                 nattempts = int(params['nattempts'][0])
 
-        result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ.get('vo'))
+        result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ['vo'])
         return Response(render_json(result), 200, content_type='application/json')
 
 
@@ -1180,7 +1180,8 @@ class BadReplicasStates(ErrorHandlingMethodView):
           406:
             description: Not acceptable
         """
-        state, rse, younger_than, older_than, limit, list_pfns = None, None, None, None, None, None
+        list_pfns = False
+        state, rse, younger_than, older_than, limit = None, None, None, None, None
         if request.query_string:
             query_string = request.query_string.decode(encoding='utf-8')
             try:
@@ -1208,7 +1209,7 @@ class BadReplicasStates(ErrorHandlingMethodView):
                                                 vo=vo):
                 yield dumps(row, cls=APIEncoder) + '\n'
 
-        return try_stream(generate(vo=request.environ.get('vo')))
+        return try_stream(generate(vo=request.environ['vo']))
 
 
 class BadReplicasSummary(ErrorHandlingMethodView):
@@ -1287,7 +1288,7 @@ class BadReplicasSummary(ErrorHandlingMethodView):
                                                 to_date=to_date, vo=vo):
                 yield dumps(row, cls=APIEncoder) + '\n'
 
-        return try_stream(generate(vo=request.environ.get('vo')))
+        return try_stream(generate(vo=request.environ['vo']))
 
 
 class DatasetReplicas(ErrorHandlingMethodView):
@@ -1370,7 +1371,7 @@ class DatasetReplicas(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
+            scope, name = parse_scope_name(scope_name, request.environ['vo'])
 
             def generate(_deep, vo):
                 for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
@@ -1378,7 +1379,7 @@ class DatasetReplicas(ErrorHandlingMethodView):
 
             deep = request.args.get('deep', default=False)
 
-            return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
+            return try_stream(generate(_deep=deep, vo=request.environ['vo']))
         except ValueError as error:
             return generate_http_error_flask(400, error)
 
@@ -1483,7 +1484,7 @@ class DatasetReplicasBulk(ErrorHandlingMethodView):
                 for row in list_dataset_replicas_bulk(dids=dids, vo=vo):
                     yield dumps(row, cls=APIEncoder) + '\n'
 
-            return try_stream(generate(vo=request.environ.get('vo')))
+            return try_stream(generate(vo=request.environ['vo']))
         except InvalidObject as error:
             return generate_http_error_flask(400, error, f'Cannot validate DIDs: {error}')
 
@@ -1520,7 +1521,7 @@ class DatasetReplicasVP(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
+            scope, name = parse_scope_name(scope_name, request.environ['vo'])
 
             def generate(_deep, vo):
                 for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
@@ -1528,7 +1529,7 @@ class DatasetReplicasVP(ErrorHandlingMethodView):
 
             deep = request.args.get('deep', default=False)
 
-            return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
+            return try_stream(generate(_deep=deep, vo=request.environ['vo']))
         except ValueError as error:
             return generate_http_error_flask(400, error)
 
@@ -1610,7 +1611,7 @@ class ReplicasRSE(ErrorHandlingMethodView):
             for row in list_datasets_per_rse(rse=rse, vo=vo):
                 yield dumps(row, cls=APIEncoder) + '\n'
 
-        return try_stream(generate(vo=request.environ.get('vo')))
+        return try_stream(generate(vo=request.environ['vo']))
 
 
 class BadDIDs(ErrorHandlingMethodView):
@@ -1676,11 +1677,11 @@ class BadDIDs(ErrorHandlingMethodView):
             not_declared_files = add_bad_dids(
                 dids=param_get(parameters, 'dids', default=[]),
                 rse=param_get(parameters, 'rse', default=None),
-                issuer=request.environ.get('issuer'),
+                issuer=request.environ['issuer'],
                 state=BadFilesStatus.BAD,
                 reason=param_get(parameters, 'reason', default=None),
                 expires_at=expires_at,
-                vo=request.environ.get('vo'),
+                vo=request.environ['vo'],
             )
         except (ValueError, InvalidType) as error:
             return generate_http_error_flask(400, ValueError.__name__, error.args[0])
@@ -1745,11 +1746,11 @@ class BadPFNs(ErrorHandlingMethodView):
         try:
             add_bad_pfns(
                 pfns=param_get(parameters, 'pfns', default=[]),
-                issuer=request.environ.get('issuer'),
+                issuer=request.environ['issuer'],
                 state=param_get(parameters, 'state', default=None),
                 reason=param_get(parameters, 'reason', default=None),
                 expires_at=expires_at,
-                vo=request.environ.get('vo'),
+                vo=request.environ['vo'],
             )
         except (ValueError, InvalidType) as error:
             return generate_http_error_flask(400, ValueError.__name__, error.args[0])
@@ -1816,8 +1817,8 @@ class Tombstone(ErrorHandlingMethodView):
                     rse=replica['rse'],
                     scope=replica['scope'],
                     name=replica['name'],
-                    issuer=request.environ.get('issuer'),
-                    vo=request.environ.get('vo'),
+                    issuer=request.environ['issuer'],
+                    vo=request.environ['vo'],
                 )
         except ReplicaNotFound as error:
             return generate_http_error_flask(404, error)

rucio/web/rest/flaskapi/v1/requests.py

@@ -13,18 +13,18 @@
 # limitations under the License.
 
 import json
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union, cast
 
 import flask
 from flask import Flask, Response
 
-from rucio.common.exception import RequestNotFound
+from rucio.common.exception import AccessDenied, RequestNotFound
 from rucio.common.utils import APIEncoder, render_json
 from rucio.core.rse import get_rses_with_attribute_value
-from rucio.db.sqla.constants import RequestState
+from rucio.db.sqla.constants import RequestState, TransferLimitDirection
 from rucio.gateway import request
 from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
-from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, parse_scope_name, response_headers, try_stream
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -178,7 +178,7 @@ class RequestGet(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            scope, name = parse_scope_name(scope_name, flask.request.environ.get('vo'))
+            scope, name = parse_scope_name(scope_name, flask.request.environ['vo'])
         except ValueError as error:
             return generate_http_error_flask(400, error)
 
@@ -187,8 +187,8 @@ class RequestGet(ErrorHandlingMethodView):
                 scope=scope,
                 name=name,
                 rse=rse,
-                issuer=flask.request.environ.get('issuer'),
-                vo=flask.request.environ.get('vo'),
+                issuer=flask.request.environ['issuer'],
+                vo=flask.request.environ['vo'],
             )
             return Response(json.dumps(request_data, cls=APIEncoder), content_type='application/json')
         except RequestNotFound as error:
@@ -343,7 +343,7 @@ class RequestHistoryGet(ErrorHandlingMethodView):
             description: Not acceptable
         """
         try:
-            scope, name = parse_scope_name(scope_name, flask.request.environ.get('vo'))
+            scope, name = parse_scope_name(scope_name, flask.request.environ['vo'])
         except ValueError as error:
             return generate_http_error_flask(400, error)
 
@@ -352,8 +352,8 @@ class RequestHistoryGet(ErrorHandlingMethodView):
                 scope=scope,
                 name=name,
                 rse=rse,
-                issuer=flask.request.environ.get('issuer'),
-                vo=flask.request.environ.get('vo'),
+                issuer=flask.request.environ['issuer'],
+                vo=flask.request.environ['vo'],
             )
             return Response(json.dumps(request_data, cls=APIEncoder), content_type='application/json')
         except RequestNotFound as error:
@@ -564,11 +564,11 @@ class RequestList(ErrorHandlingMethodView):
         src_rses = []
         dst_rses = []
         if src_site:
-            src_rses = get_rses_with_attribute_value(key='site', value=src_site, vo=flask.request.environ.get('vo'))
+            src_rses = get_rses_with_attribute_value(key='site', value=src_site, vo=flask.request.environ['vo'])
             if not src_rses:
                 return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {src_site} to RSE')
             src_rses = [rse['rse_name'] for rse in src_rses]
-            dst_rses = get_rses_with_attribute_value(key='site', value=dst_site, vo=flask.request.environ.get('vo'))
+            dst_rses = get_rses_with_attribute_value(key='site', value=dst_site, vo=flask.request.environ['vo'])
             if not dst_rses:
                 return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {dst_site} to RSE')
             dst_rses = [rse['rse_name'] for rse in dst_rses]
@@ -576,11 +576,15 @@ class RequestList(ErrorHandlingMethodView):
             dst_rses = [dst_rse]
             src_rses = [src_rse]
 
+        # Manual cast to list[str] as static code analysis erroneously sees these as list[Optional[str]]
+        src_rses = cast("list[str]", src_rses)
+        dst_rses = cast("list[str]", dst_rses)
+
         def generate(issuer, vo):
             for result in request.list_requests(src_rses, dst_rses, states, issuer=issuer, vo=vo):
                 yield render_json(**result) + '\n'
 
-        return try_stream(generate(issuer=flask.request.environ.get('issuer'), vo=flask.request.environ.get('vo')))
+        return try_stream(generate(issuer=flask.request.environ['issuer'], vo=flask.request.environ['vo']))
 
 
 class RequestHistoryList(ErrorHandlingMethodView):
@@ -779,8 +783,8 @@ class RequestHistoryList(ErrorHandlingMethodView):
         src_site = flask.request.args.get('src_site', default=None)
         dst_site = flask.request.args.get('dst_site', default=None)
         request_states = flask.request.args.get('request_states', default=None)
-        offset = flask.request.args.get('offset', default=0)
-        limit = flask.request.args.get('limit', default=100)
+        offset = flask.request.args.get('offset', type=int, default=0)
+        limit = flask.request.args.get('limit', type=int, default=100)
 
         if not request_states:
             return generate_http_error_flask(400, 'MissingParameter', 'Request state is missing')
@@ -801,11 +805,11 @@ class RequestHistoryList(ErrorHandlingMethodView):
         src_rses = []
         dst_rses = []
         if src_site:
-            src_rses = get_rses_with_attribute_value(key='site', value=src_site, vo=flask.request.environ.get('vo'))
+            src_rses = get_rses_with_attribute_value(key='site', value=src_site, vo=flask.request.environ['vo'])
             if not src_rses:
                 return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {src_site} to RSE')
             src_rses = [rse['rse_name'] for rse in src_rses]
-            dst_rses = get_rses_with_attribute_value(key='site', value=dst_site, vo=flask.request.environ.get('vo'))
+            dst_rses = get_rses_with_attribute_value(key='site', value=dst_site, vo=flask.request.environ['vo'])
             if not dst_rses:
                 return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {dst_site} to RSE')
             dst_rses = [rse['rse_name'] for rse in dst_rses]
@@ -813,11 +817,15 @@ class RequestHistoryList(ErrorHandlingMethodView):
             dst_rses = [dst_rse]
             src_rses = [src_rse]
 
+        # Manual cast to list[str] as static code analysis erroneously sees these as list[Optional[str]]
+        src_rses = cast("list[str]", src_rses)
+        dst_rses = cast("list[str]", dst_rses)
+
         def generate(issuer, vo):
             for result in request.list_requests_history(src_rses, dst_rses, states, issuer=issuer, vo=vo, offset=offset, limit=limit):
                 yield render_json(**result) + '\n'
 
-        return try_stream(generate(issuer=flask.request.environ.get('issuer'), vo=flask.request.environ.get('vo')))
+        return try_stream(generate(issuer=flask.request.environ['issuer'], vo=flask.request.environ['vo']))
 
 
 class RequestMetricsGet(ErrorHandlingMethodView):
@@ -960,8 +968,8 @@ class RequestMetricsGet(ErrorHandlingMethodView):
             src_rse=src_rse,
             activity=activity,
             group_by_rse_attribute=group_by_rse_attribute,
-            issuer=flask.request.environ.get('issuer'),
-            vo=flask.request.environ.get('vo')
+            issuer=flask.request.environ['issuer'],
+            vo=flask.request.environ['vo']
         )
 
         if format == 'panda':
@@ -972,6 +980,187 @@ class RequestMetricsGet(ErrorHandlingMethodView):
                 yield render_json(**result) + '\n'
         return try_stream(generate())
 
+class TransferLimits(ErrorHandlingMethodView):
+    """ REST API to get, set or delete transfer limits. """
+
+    @check_accept_header_wrapper_flask(['application/x-json-stream'])
+    def get(self) -> flask.Response:
+        """
+        ---
+        summary: Get Transfer Limits
+        description: Get all the transfer limits.
+        tags:
+          - Requests
+        responses:
+          200:
+            description: OK
+            content:
+              application/x-json-stream:
+                schema:
+                  description: All the transfer limits
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      id:
+                        description: The transfer limit id.
+                        type: string
+                      rse_expression:
+                        description: The RSE expression for which the limit applies.
+                        type: string
+                      direction:
+                        description: The direction in which this limit applies (source/destination)
+                        type: string
+                      max_transfers:
+                        description: Maximum number of transfers allowed.
+                        type: integer
+                      volume:
+                        description: Maximum transfer volume in bytes.
+                        type: integer
+                      deadline:
+                        description: Maximum waiting time in hours until a datasets gets released.
+                        type: integer
+                      strategy:
+                        description: defines how to handle datasets: `fifo` (each file released separately) or `grouped_fifo` (wait for the entire dataset to fit)
+                        type: string
+                      transfers:
+                        description: Current number of active transfers
+                        type: integer
+                      waitings:
+                        description: Current number of waiting transfers
+                        type: integer
+                      updated_at:
+                        description: Datetime of the last update.
+                        type: string
+                      created_at:
+                        description: Datetime of the creation of the transfer limit.
+                        type: string
+          401:
+            description: Invalid Auth Token
+        """
+        transfer_limits = request.list_transfer_limits(issuer=flask.request.environ['issuer'], vo=flask.request.environ['vo'])
+        def generate() -> "Iterator[str]":
+                for limit in transfer_limits:
+                    yield json.dumps(limit, cls=APIEncoder) + '\n'
+        return try_stream(generate())
+
+    def put(self) -> Union[flask.Response, tuple[str, int]]:
+        """
+        ---
+        summary: Set Transfer Limit
+        description: Create or update a transfer limit for a specific RSE expression and activity.
+        tags:
+          - Requests
+        requestBody:
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - rse_expression
+                  - max_transfers
+                properties:
+                  rse_expression:
+                    type: string
+                    description: The RSE expression for which the transfer limit is being set.
+                  activity:
+                    type: string
+                    description: The activity to which the transfer limit applies.
+                  max_transfers:
+                    type: integer
+                    description: The maximum number of transfers allowed.
+                  direction:
+                    type: string
+                    description: The direction of the transfer limit (source or destination).
+                    enum: ["SOURCE", "DESTINATION"]
+                    default: "DESTINATION"
+                  volume:
+                    type: integer
+                    description: The maximum transfer volume in bytes.
+                  deadline:
+                    type: integer
+                    description: The maximum waiting time in hours until a dataset is released.
+                  strategy:
+                    type: string
+                    description: The strategy for handling datasets (e.g., `fifo` or `grouped_fifo`).
+                  transfers:
+                    type: integer
+                    description: The current number of active transfers.
+                  waitings:
+                    type: integer
+                    description: The current number of waiting transfers.
+        responses:
+          201:
+            description: Transfer limit set successfully.
+          400:
+            description: Invalid input data.
+          401:
+            description: Invalid Auth Token.
+          500:
+            description: Internal server error.
+        """
+        parameters = json_parameters()
+        rse_expression = param_get(parameters, 'rse_expression')
+        max_transfers = param_get(parameters, 'max_transfers')
+
+        try:
+          request.set_transfer_limit(
+            rse_expression=rse_expression,
+            max_transfers=max_transfers,
+            activity=param_get(parameters, 'activity', default=None),
+            direction=param_get(parameters, 'direction', default=TransferLimitDirection.DESTINATION),
+            volume=param_get(parameters, 'volume', default=None),
+            deadline=param_get(parameters, 'deadline', default=None),
+            strategy=param_get(parameters, 'strategy', default=None),
+            transfers=param_get(parameters, 'transfers', default=None),
+            waitings=param_get(parameters, 'waitings', default=None),
+            issuer=flask.request.environ['issuer'],
+            vo=flask.request.environ['vo']
+          )
+        except AccessDenied as error:
+          return generate_http_error_flask(401, error)
+
+        return '', 201
+
+    def delete(self) -> Union[flask.Response, tuple[str, int]]:
+        """
+        ---
+        summary: Delete Transfer Limit
+        description: Delete a transfer limit for an RSE expression.
+        tags:
+          - Requests
+        parameters:
+          - name: rse_expression
+            in: query
+            description: The RSE expression to delete the limit for.
+            required: true
+            schema:
+              type: string
+        responses:
+          200:
+            description: Transfer limit deleted successfully.
+          400:
+            description: Invalid input data.
+          401:
+            description: Invalid Auth Token.
+          500:
+            description: Internal server error.
+        """
+        parameters = json_parameters()
+        rse_expression = param_get(parameters, 'rse_expression')
+
+        try:
+          request.delete_transfer_limit(
+            rse_expression=rse_expression,
+            activity=param_get(parameters, 'activity', default=None),
+            direction=param_get(parameters, 'direction', default=TransferLimitDirection.DESTINATION),
+            issuer=flask.request.environ['issuer'],
+            vo=flask.request.environ['vo']
+          )
+        except AccessDenied as error:
+          return generate_http_error_flask(401, error)
+
+        return '', 200
 
 def blueprint():
     bp = AuthenticatedBlueprint('requests', __name__, url_prefix='/requests')
@@ -986,6 +1175,8 @@ def blueprint():
     bp.add_url_rule('/history/list', view_func=request_history_list_view, methods=['get', ])
     request_metrics_view = RequestMetricsGet.as_view('request_metrics_get')
     bp.add_url_rule('/metrics', view_func=request_metrics_view, methods=['get', ])
+    transfer_limits_view = TransferLimits.as_view('transfer_limits_get')
+    bp.add_url_rule('/transfer_limits', view_func=transfer_limits_view, methods=['get', 'put', 'delete'])
 
     bp.after_request(response_headers)
     return bp