rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

rucio/web/rest/flaskapi/v1/requests.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +13,21 @@
 # limitations under the License.
 
 import json
+from typing import TYPE_CHECKING
 
 import flask
 from flask import Flask, Response
 
-from rucio.api import request
 from rucio.common.exception import RequestNotFound
 from rucio.common.utils import APIEncoder, render_json
 from rucio.core.rse import get_rses_with_attribute_value
 from rucio.db.sqla.constants import RequestState
+from rucio.gateway import request
 from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
-from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \
-    response_headers, generate_http_error_flask, ErrorHandlingMethodView
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, parse_scope_name, response_headers, try_stream
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
 
 
 class RequestGet(ErrorHandlingMethodView):
@@ -36,7 +38,7 @@ class RequestGet(ErrorHandlingMethodView):
         """
         ---
         summary: Get Request
-        description: Get a request for a given DID to a destinaion RSE.
+        description: Get a request for a given DID to a destination RSE.
         tags:
           - Requests
         parameters:
@@ -100,7 +102,7 @@ class RequestGet(ErrorHandlingMethodView):
                       description: The numbers of attempted retires.
                       type: integer
                     err_msg:
-                      description: An error message if one occured.
+                      description: An error message if one occurred.
                       type: string
                     previous_attempt_id:
                       description: The id of the previous attempt.
@@ -130,7 +132,7 @@ class RequestGet(ErrorHandlingMethodView):
                       description: The time the request got started.
                       type: string
                     transferred_at:
-                      description: The time the request got transfered.
+                      description: The time the request got transferred.
                       type: string
                     estimated_at:
                       description: The time the request got estimated.
@@ -142,7 +144,7 @@ class RequestGet(ErrorHandlingMethodView):
                       description: The estimation of the started at value.
                       type: string
                     estimated_transferred_at:
-                      description: The estimation of the transfered at value.
+                      description: The estimation of the transferred at value.
                       type: string
                     staging_started_at:
                       description: The time the staging got started.
@@ -201,7 +203,7 @@ class RequestHistoryGet(ErrorHandlingMethodView):
         """
         ---
         summary: Get Historical Request
-        description: List a hostorical request for a given DID to a destination RSE.
+        description: List a historical request for a given DID to a destination RSE.
         tags:
           - Requests
         parameters:
@@ -265,7 +267,7 @@ class RequestHistoryGet(ErrorHandlingMethodView):
                       description: The numbers of attempted retires.
                       type: integer
                     err_msg:
-                      description: An error message if one occured.
+                      description: An error message if one occurred.
                       type: string
                     previous_attempt_id:
                       description: The id of the previous attempt.
@@ -295,7 +297,7 @@ class RequestHistoryGet(ErrorHandlingMethodView):
                       description: The time the request got started.
                       type: string
                     transferred_at:
-                      description: The time the request got transfered.
+                      description: The time the request got transferred.
                       type: string
                     estimated_at:
                       description: The time the request got estimated.
@@ -307,7 +309,7 @@ class RequestHistoryGet(ErrorHandlingMethodView):
                       description: The estimation of the started at value.
                       type: string
                     estimated_transferred_at:
-                      description: The estimation of the transfered at value.
+                      description: The estimation of the transferred at value.
                       type: string
                     staging_started_at:
                       description: The time the staging got started.
@@ -365,7 +367,7 @@ class RequestList(ErrorHandlingMethodView):
     def get(self):
         """
         ---
-        summary: List Historic Requests
+        summary: List Requests
         description: List requests for a given source and destination RSE or site.
         tags:
           - Requests
@@ -417,7 +419,7 @@ class RequestList(ErrorHandlingMethodView):
             content:
               application/x-json-stream:
                 schema:
-                  description: All requests matching the arguments. Seperated by the new line character.
+                  description: All requests matching the arguments. Separated by the new line character.
                   type: array
                   items:
                     description: A request.
@@ -462,7 +464,7 @@ class RequestList(ErrorHandlingMethodView):
                         description: The numbers of attempted retires.
                         type: integer
                       err_msg:
-                        description: An error message if one occured.
+                        description: An error message if one occurred.
                         type: string
                       previous_attempt_id:
                         description: The id of the previous attempt.
@@ -492,7 +494,7 @@ class RequestList(ErrorHandlingMethodView):
                         description: The time the request got started.
                         type: string
                       transferred_at:
-                        description: The time the request got transfered.
+                        description: The time the request got transferred.
                         type: string
                       estimated_at:
                         description: The time the request got estimated.
@@ -504,7 +506,7 @@ class RequestList(ErrorHandlingMethodView):
                         description: The estimation of the started at value.
                         type: string
                       estimated_transferred_at:
-                        description: The estimation of the transfered at value.
+                        description: The estimation of the transferred at value.
                         type: string
                       staging_started_at:
                         description: The time the staging got started.
@@ -652,7 +654,7 @@ class RequestHistoryList(ErrorHandlingMethodView):
             content:
               application/x-json-stream:
                 schema:
-                  description: All requests matching the arguments. Seperated by a new line character.
+                  description: All requests matching the arguments. Separated by a new line character.
                   type: array
                   items:
                     description: A request.
@@ -697,7 +699,7 @@ class RequestHistoryList(ErrorHandlingMethodView):
                         description: The numbers of attempted retires.
                         type: integer
                       err_msg:
-                        description: An error message if one occured.
+                        description: An error message if one occurred.
                         type: string
                       previous_attempt_id:
                         description: The id of the previous attempt.
@@ -727,7 +729,7 @@ class RequestHistoryList(ErrorHandlingMethodView):
                         description: The time the request got started.
                         type: string
                       transferred_at:
-                        description: The time the request got transfered.
+                        description: The time the request got transferred.
                         type: string
                       estimated_at:
                         description: The time the request got estimated.
@@ -739,7 +741,7 @@ class RequestHistoryList(ErrorHandlingMethodView):
                         description: The estimation of the started at value.
                         type: string
                       estimated_transferred_at:
-                        description: The estimation of the transfered at value.
+                        description: The estimation of the transferred at value.
                         type: string
                       staging_started_at:
                         description: The time the staging got started.
@@ -818,6 +820,159 @@ class RequestHistoryList(ErrorHandlingMethodView):
         return try_stream(generate(issuer=flask.request.environ.get('issuer'), vo=flask.request.environ.get('vo')))
 
 
+class RequestMetricsGet(ErrorHandlingMethodView):
+    """ REST API to get request stats. """
+
+    @check_accept_header_wrapper_flask(['application/x-json-stream'])
+    def get(self):
+        """
+        ---
+        summary: Get Request Statistics
+        description: Get statistics of requests grouped by source, destination, and activity.
+        tags:
+          - Requests
+        parameters:
+        - name: dest_rse
+          in: query
+          description: The destination RSE name
+          schema:
+            type: string
+        - name: source_rse
+          in: query
+          description: The source RSE name
+          schema:
+            type: string
+        - name: activity
+          in: query
+          description: The activity
+          schema:
+            type: string
+        - name: group_by_rse_attribute
+          in: query
+          description: The parameter to group the RSEs by.
+          schema:
+            type: string
+        responses:
+          200:
+            description: OK
+            content:
+              application/x-json-stream:
+                schema:
+                  description: Statistics of requests by source, destination, and activity.
+                  type: array
+                  items:
+                    description: Statistics of the request group for a given (source, destination, activity) tuple.
+                    type: object
+                    properties:
+                      src_rse:
+                        type: string
+                        description: The name of this links source RSE
+                      dst_rse:
+                        type: string
+                        description: The name of this links destination RSE
+                      distance:
+                        type: integer
+                        description: The distance between the source and destination RSE
+                      files:
+                        type: object
+                        properties:
+                          done-total-1h:
+                            type: integer
+                            description: The total number of files successfully transferred in the last 1 hour
+                          done-total-6h:
+                            type: integer
+                            description: The total number of files successfully transferred in the last 6 hours
+                          failed-total-1h:
+                            type: integer
+                            description: The total number of transfer failures in the last 1 hour
+                          failed-total-6h:
+                            type: integer
+                            description: The total number of transfer failures in the last 6 hours
+                          queued-total:
+                            type: integer
+                            description: The total number of files queued in rucio
+                          queued:
+                            type: object
+                            description: Per-activity number of queued files
+                            additionalProperties:
+                              type: integer
+                          done:
+                            type: object
+                            additionalProperties:
+                              type: object
+                              properties:
+                                1h:
+                                  type: integer
+                                6h:
+                                  type: integer
+                          failed:
+                            type: object
+                            description: Per-activity number of transfer failures in the last 1 and 6 hours
+                            additionalProperties:
+                              type: object
+                              properties:
+                                1h:
+                                  type: integer
+                                6h:
+                                  type: integer
+                      bytes:
+                        type: object
+                        properties:
+                          done-total-1h:
+                            type: integer
+                            description: The total number of bytes successfully transferred in the last 1 hour
+                          done-total-6h:
+                            type: integer
+                            description: The total number of bytes successfully transferred in the last 6 hours
+                          queued-total:
+                            type: integer
+                            description: The total number of bytes queued to be transferred by rucio
+                          queued:
+                            type: object
+                            description: Per-activity amount of queued bytes
+                            additionalProperties:
+                              type: integer
+                          done:
+                            type: object
+                            description: Per-activity number of transferred bytes in the last 1 and 6 hours
+                            additionalProperties:
+                              type: object
+                              properties:
+                                1h:
+                                  type: integer
+                                6h:
+                                  type: integer
+                    required:
+                      - distance
+                      - src_rse
+                      - dst_rse
+          401:
+            description: Invalid Auth Token
+        """
+        dst_rse = flask.request.args.get('dst_rse', default=None)
+        src_rse = flask.request.args.get('src_rse', default=None)
+        activity = flask.request.args.get('activity', default=None)
+        group_by_rse_attribute = flask.request.args.get('group_by_rse_attribute', default=None)
+        format = flask.request.args.get('format', default=None)
+
+        metrics = request.get_request_metrics(
+            dst_rse=dst_rse,
+            src_rse=src_rse,
+            activity=activity,
+            group_by_rse_attribute=group_by_rse_attribute,
+            issuer=flask.request.environ.get('issuer'),
+            vo=flask.request.environ.get('vo')
+        )
+
+        if format == 'panda':
+            return Response(json.dumps(metrics, cls=APIEncoder), content_type='application/json')
+
+        def generate() -> "Iterator[str]":
+            for result in metrics.values():
+                yield render_json(**result) + '\n'
+        return try_stream(generate())
+
+
 def blueprint():
     bp = AuthenticatedBlueprint('requests', __name__, url_prefix='/requests')
 
@@ -829,6 +984,8 @@ def blueprint():
     bp.add_url_rule('/list', view_func=request_list_view, methods=['get', ])
     request_history_list_view = RequestHistoryList.as_view('request_history_list')
     bp.add_url_rule('/history/list', view_func=request_history_list_view, methods=['get', ])
+    request_metrics_view = RequestMetricsGet.as_view('request_metrics_get')
+    bp.add_url_rule('/metrics', view_func=request_metrics_view, methods=['get', ])
 
     bp.after_request(response_headers)
     return bp

rucio/web/rest/flaskapi/v1/rses.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,21 +14,57 @@
 
 from json import dumps
 
-from flask import Flask, Response, request, jsonify
-
-from rucio.api.account_limit import get_rse_account_usage
-from rucio.api.rse import add_rse, update_rse, list_rses, del_rse, add_rse_attribute, list_rse_attributes, \
-    del_rse_attribute, add_protocol, get_rse_protocols, del_protocols, update_protocols, get_rse, set_rse_usage, \
-    get_rse_usage, list_rse_usage_history, set_rse_limits, get_rse_limits, delete_rse_limits, parse_rse_expression, \
-    add_distance, get_distance, update_distance, delete_distance, list_qos_policies, add_qos_policy, delete_qos_policy
-from rucio.common.exception import Duplicate, AccessDenied, RSENotFound, RSEOperationNotSupported, \
-    RSEProtocolNotSupported, InvalidObject, RSEProtocolDomainNotSupported, RSEProtocolPriorityError, \
-    InvalidRSEExpression, RSEAttributeNotFound, CounterNotFound, InvalidPath, ReplicaNotFound, InputValidationError
-from rucio.common.utils import Availability, render_json, APIEncoder
+from flask import Flask, Response, jsonify, request
+
+from rucio.common.exception import (
+    AccessDenied,
+    CounterNotFound,
+    Duplicate,
+    InputValidationError,
+    InvalidObject,
+    InvalidPath,
+    InvalidRSEExpression,
+    ReplicaNotFound,
+    RSEAttributeNotFound,
+    RSENotFound,
+    RSEOperationNotSupported,
+    RSEProtocolDomainNotSupported,
+    RSEProtocolNotSupported,
+    RSEProtocolPriorityError,
+)
+from rucio.common.utils import APIEncoder, Availability, render_json
+from rucio.gateway.account_limit import get_rse_account_usage
+from rucio.gateway.rse import (
+    add_distance,
+    add_protocol,
+    add_qos_policy,
+    add_rse,
+    add_rse_attribute,
+    del_protocols,
+    del_rse,
+    del_rse_attribute,
+    delete_distance,
+    delete_qos_policy,
+    delete_rse_limits,
+    get_distance,
+    get_rse,
+    get_rse_limits,
+    get_rse_protocols,
+    get_rse_usage,
+    list_qos_policies,
+    list_rse_attributes,
+    list_rse_usage_history,
+    list_rses,
+    parse_rse_expression,
+    set_rse_limits,
+    set_rse_usage,
+    update_distance,
+    update_protocols,
+    update_rse,
+)
 from rucio.rse import rsemanager
 from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
-from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \
-    try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream
 
 
 class RSEs(ErrorHandlingMethodView):
@@ -205,8 +240,8 @@ class RSE(ErrorHandlingMethodView):
                     description: The rse type.
                     type: string
                     enum: ["DISK", "TAPE"]
-                  latitute:
-                    description: The latitute of the RSE.
+                  latitude:
+                    description: The latitude of the RSE.
                     type: number
                   longitude:
                     description: The longitude of the RSE.
@@ -346,8 +381,8 @@ class RSE(ErrorHandlingMethodView):
                     description: The rse type.
                     type: string
                     enum: ["DISK", "TAPE"]
-                  latitute:
-                    description: The latitute of the RSE.
+                  latitude:
+                    description: The latitude of the RSE.
                     type: number
                   longitude:
                     description: The longitude of the RSE.
@@ -440,8 +475,8 @@ class RSE(ErrorHandlingMethodView):
                       description: The rse type.
                       type: string
                       enum: ["DISK", "TAPE"]
-                    latitute:
-                      description: The latitute of the RSE.
+                    latitude:
+                      description: The latitude of the RSE.
                       type: number
                     longitude:
                       description: The longitude of the RSE.
@@ -787,7 +822,7 @@ class ProtocolList(ErrorHandlingMethodView):
           401:
             description: Invalid Auth Token
           404:
-            description: RSE not found or RSE Operation, RSE Protocal Doman, RSE Protocol not supported
+            description: RSE not found or RSE Operation, RSE Protocol Domain, RSE Protocol not supported
           406:
             description: Not acceptable
         """
@@ -838,7 +873,7 @@ class LFNS2PFNS(ErrorHandlingMethodView):
             type: string
         - name: operation
           in: query
-          description: Optional query argument to select the protoco for read-vs-writes.
+          description: Optional query argument to select the protocol for read-vs-writes.
           schema:
             type: string
         responses:
@@ -855,7 +890,7 @@ class LFNS2PFNS(ErrorHandlingMethodView):
           401:
             description: Invalid Auth Token
           404:
-            description: RSE not found or RSE Protocol or RSE Protocl Domain not supported
+            description: RSE not found or RSE Protocol or RSE Protocol Domain not supported
           406:
             description: Not acceptable
         """
@@ -1315,7 +1350,7 @@ class Protocol(ErrorHandlingMethodView):
         """
         ---
         summary: Delete Protocol Attributes
-        description: Delete all protocol attibutes.
+        description: Delete all protocol attributes.
         tags:
           - Rucio Storage Elements
         parameters:
@@ -1813,7 +1848,7 @@ class Distance(ErrorHandlingMethodView):
                   type: array
                   items:
                     type: object
-                    description: One distance betweeen source and destination.
+                    description: One distance between source and destination.
                     properties:
                       src_rse_id:
                         description: The source rse id.
@@ -2140,7 +2175,7 @@ class QoSPolicy(ErrorHandlingMethodView):
                   type: array
                   items:
                     type: object
-                    porperties:
+                    properties:
                       rse_id:
                         description: The rse id.
                         type: string

rucio/web/rest/flaskapi/v1/rules.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,20 +15,44 @@
 from json import dumps
 from typing import Any
 
-from flask import Flask, request, Response
-
-from rucio.api.lock import get_replica_locks_for_rule_id
-from rucio.api.rule import add_replication_rule, delete_replication_rule, get_replication_rule, \
-    update_replication_rule, reduce_replication_rule, list_replication_rule_history, \
-    list_replication_rule_full_history, list_replication_rules, examine_replication_rule, move_replication_rule
-from rucio.common.exception import InputValidationError, InsufficientAccountLimit, RuleNotFound, AccessDenied, InvalidRSEExpression, \
-    InvalidReplicationRule, DataIdentifierNotFound, InsufficientTargetRSEs, ReplicationRuleCreationTemporaryFailed, \
-    InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule, InvalidObject, AccountNotFound, \
-    RuleReplaceFailed, ScratchDiskLifetimeConflict, ManualRuleApprovalBlocked, UnsupportedOperation
-from rucio.common.utils import render_json, APIEncoder
+from flask import Flask, Response, request
+
+from rucio.common.exception import (
+    AccessDenied,
+    AccountNotFound,
+    DataIdentifierNotFound,
+    DuplicateRule,
+    InputValidationError,
+    InsufficientAccountLimit,
+    InsufficientTargetRSEs,
+    InvalidObject,
+    InvalidReplicationRule,
+    InvalidRSEExpression,
+    InvalidRuleWeight,
+    ManualRuleApprovalBlocked,
+    ReplicationRuleCreationTemporaryFailed,
+    RuleNotFound,
+    RuleReplaceFailed,
+    ScratchDiskLifetimeConflict,
+    StagingAreaRuleRequiresLifetime,
+    UnsupportedOperation,
+)
+from rucio.common.utils import APIEncoder, render_json
+from rucio.gateway.lock import get_replica_locks_for_rule_id
+from rucio.gateway.rule import (
+    add_replication_rule,
+    delete_replication_rule,
+    examine_replication_rule,
+    get_replication_rule,
+    list_replication_rule_full_history,
+    list_replication_rule_history,
+    list_replication_rules,
+    move_replication_rule,
+    reduce_replication_rule,
+    update_replication_rule,
+)
 from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
-from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \
-    response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream
 
 
 class Rule(ErrorHandlingMethodView):
@@ -253,7 +276,14 @@ class AllRule(ErrorHandlingMethodView):
                     description: The list of data identifiers.
                     type: array
                     items:
-                      type: string
+                      type: object
+                      properties:
+                        scope:
+                          description: The scope of the data identifier
+                          type: string
+                        name:
+                          description: The name of the data identifier
+                          type: string
                   account:
                     description: The account of the issuer.
                     type: string
@@ -763,16 +793,16 @@ class RuleAnalysis(ErrorHandlingMethodView):
                             description: The name of the lock.
                           rse_id:
                             type: string
-                            description: The rse_id of the transfered lock.
+                            description: The rse_id of the transferred lock.
                           rse:
                             type: object
-                            description: Information about the rse of the transfered lock.
+                            description: Information about the rse of the transferred lock.
                           attempts:
                             type: integer
                             description: The number of attempts.
                           last_error:
                             type: string
-                            description: The last error that occured.
+                            description: The last error that occurred.
                           last_source:
                             type: string
                             description: The last source.

rucio/web/rest/flaskapi/v1/scopes.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from flask import Flask, request, jsonify
+from flask import Flask, jsonify, request
 
-from rucio.api.scope import add_scope, list_scopes, get_scopes
 from rucio.common.exception import AccountNotFound, Duplicate, ScopeNotFound
+from rucio.gateway.scope import add_scope, get_scopes, list_scopes
 from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
-from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, response_headers, \
-    generate_http_error_flask, ErrorHandlingMethodView
+from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, response_headers
 
 
 class Scope(ErrorHandlingMethodView):