panda-server-source 0.8.0__py3-none-any.whl

pandaserver/api/v1/statistics_api.py

@@ -0,0 +1,207 @@
+from typing import Any, Dict
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.api.v1.common import generate_response, request_validation
+from pandaserver.srvcore.panda_request import PandaRequest
+from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
+
+_logger = PandaLogger().getLogger("api_statistics")
+
+global_task_buffer = None
+
+MAX_TIME_WINDOW = 60 * 24 * 7  # 7 days
+
+
+def init_task_buffer(task_buffer: TaskBuffer) -> None:
+    """
+    Initialize the task buffer. This method needs to be called before any other method in this module.
+    """
+    global global_task_buffer
+    global_task_buffer = task_buffer
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def job_stats_by_cloud(req: PandaRequest, type: str = "production") -> Dict[str, Any]:
+    """
+    Job statistics by cloud.
+
+    Get the job statistics by cloud, which includes the active jobs and jobs in final states modified in the last 12 hours. You have to filter the statistics by type, which can be either "production" or "analysis". Used by panglia monitoring. Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/statistics/job_stats_by_cloud
+
+    Args:
+        req(PandaRequest): internally generated request object
+        type(str): can be "analysis" or "production". Defaults to "production" when not provided.
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the job statistics by cloud. When unsuccessful, the message field contains the error message.
+    """
+    tmp_logger = LogWrapper(_logger, f"job_stats_by_cloud < type={type} >")
+
+    tmp_logger.debug("Start")
+
+    if type not in ["production", "analysis"]:
+        tmp_logger.error("Invalid type parameter")
+        return generate_response(False, 'Parameter "type" needs to be either "production" or "analysis" ', {})
+
+    data = global_task_buffer.getJobStatisticsForExtIF(type)
+    success = data != {}
+    message = "" if success else "Database failure getting the job statistics"
+    tmp_logger.debug(f"Done. {message}")
+
+    return generate_response(success, message, data)
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def production_job_stats_by_cloud_and_processing_type(req: PandaRequest) -> Dict[str, Any]:
+    """
+    Production job statistics by cloud and processing type.
+
+    Get the production job statistics by cloud and processing type, which includes the active jobs and jobs in final states modified in the last 12 hours. Used by panglia monitoring. Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/statistics/production_job_stats_by_cloud_and_processing_type
+
+    Args:
+        req(PandaRequest): internally generated request object
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the job statistics by cloud. When unsuccessful, the message field contains the error message.
+    """
+
+    tmp_logger = LogWrapper(_logger, "production_job_stats_by_cloud_and_processing_type")
+
+    tmp_logger.debug("Start")
+    data = global_task_buffer.getJobStatisticsForBamboo()
+    success = data != {}
+    message = "" if success else "Database failure getting the job statistics"
+    tmp_logger.debug(f"Done. {message}")
+
+    return generate_response(success, message, data)
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def active_job_stats_by_site(req: PandaRequest) -> Dict[str, Any]:
+    """
+    Active job statistics by site
+
+    Get the active (not in a final state) job statistics by site. Used by Harvester. Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/statistics/active_job_stats_by_site
+
+    Args:
+        req(PandaRequest): internally generated request object
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the job statistics by cloud. When unsuccessful, the message field contains the error message.
+    """
+    tmp_logger = LogWrapper(_logger, "active_job_stats_by_site")
+
+    tmp_logger.debug("Start")
+    data = global_task_buffer.getJobStatistics()
+    success = data != {}
+    message = "" if success else "Database failure getting the job statistics"
+    tmp_logger.debug(f"Done. {message}")
+
+    return generate_response(success, message, data)
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def active_job_detailed_stats_by_site(req: PandaRequest) -> Dict[str, Any]:
+    """
+    Active job detailed statistics by site, resource_type and prodsourcelabel
+
+    Get the active (not in a final state) job statistics by site, resource_type and prodsourcelabel. Used by Harvester.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/statistics/active_job_detailed_stats_by_site
+
+    Args:
+        req(PandaRequest): internally generated request object
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the job statistics by site, resource_type and prodsourcelabel. When unsuccessful, the message field contains the error message.
+    """
+    tmp_logger = LogWrapper(_logger, "active_job_detailed_stats_by_site")
+
+    tmp_logger.debug("Start")
+    data = global_task_buffer.getDetailedJobStatistics()
+    success = data != {}
+    message = "" if success else "Database failure getting the job statistics with resource_type and prodsourcelabel"
+    tmp_logger.debug(f"Done. {message}")
+
+    return generate_response(success, message, data)
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def job_stats_by_site_and_resource_type(req: PandaRequest, time_window: int = None) -> Dict[str, Any]:
+    """
+    Job statistics by site and resource type
+
+    Get the job statistics by computing site (PanDA queue) and resource type (SCORE, MCORE, ...). This includes the active jobs and jobs in final states modified in the specified time window (default of 12 hours). Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/statistics/job_stats_by_site_and_resource_type
+
+    Args:
+        req(PandaRequest): internally generated request object
+        time_window(int, optional): time window in minutes for the statistics (affects only archived jobs)
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the job statistics by cloud. When unsuccessful, the message field contains the error message.
+    """
+    tmp_logger = LogWrapper(_logger, f"job_stats_by_site_and_resource_type < time_window={time_window} >")
+
+    tmp_logger.debug("Start")
+    if time_window is not None and (not isinstance(time_window, int) or time_window < 0 or time_window > MAX_TIME_WINDOW):
+        tmp_logger.error("Invalid time_window parameter")
+        return generate_response(False, 'Parameter "time_window" needs to be a positive integer smaller than 10080 min (7 days)', {})
+
+    data = global_task_buffer.getJobStatisticsPerSiteResource(time_window)
+    success = data != {}
+    message = "" if success else "Database failure getting the job statistics"
+    tmp_logger.debug(f"Done. {message}")
+
+    return generate_response(success, message, data)
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def job_stats_by_site_share_and_resource_type(req: PandaRequest, time_window: int = None) -> Dict[str, Any]:
+    """
+    Job statistics by site, global share and resource type
+
+    Get the job statistics by computing site (PanDA queue), global share and resource type (SCORE, MCORE, ...). This includes the active jobs and jobs in final states modified in the specified time window (default of 12 hours). Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/statistics/job_stats_by_site_share_and_resource_type
+
+    Args:
+        req(PandaRequest): internally generated request object
+        time_window(int): time window in minutes for the statistics (affects only archived jobs)
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the job statistics by cloud. When unsuccessful, the message field contains the error message.
+    """
+    tmp_logger = LogWrapper(_logger, f"job_stats_by_site_share_and_resource_type < time_window={time_window} >")
+
+    tmp_logger.debug("Start")
+    if time_window is not None and (not isinstance(time_window, int) or time_window < 0 or time_window > MAX_TIME_WINDOW):
+        tmp_logger.error("Invalid time_window parameter")
+        return generate_response(False, 'Parameter "time_window" needs to be a positive integer smaller than 10080 min (7 days)', {})
+
+    data = global_task_buffer.get_job_statistics_per_site_label_resource(time_window)
+    success = data != {}
+    message = "" if success else "Database failure getting the job statistics"
+    tmp_logger.debug(f"Done. {message}")
+
+    return generate_response(success, message, data)

pandaserver/api/v1/system_api.py

@@ -0,0 +1,186 @@
+from typing import Dict
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.api.v1.common import (
+    extract_primary_production_working_group,
+    extract_production_working_groups,
+    generate_response,
+    get_dn,
+    get_email_address,
+    get_fqan,
+    has_production_role,
+    request_validation,
+)
+from pandaserver.config import panda_config
+from pandaserver.srvcore.CoreUtils import clean_user_id
+from pandaserver.srvcore.panda_request import PandaRequest
+
+_logger = PandaLogger().getLogger("api_system")
+
+
+@request_validation(_logger, secure=True, request_method="GET")
+def get_attributes(req: PandaRequest, **kwargs: dict) -> Dict:
+    """
+    Get attributes
+
+    Gets all parameters and environment variables.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/system/get_attributes
+
+    Args:
+        req(PandaRequest): internally generated request object containing the env variables
+        **kwargs(dict): arbitrary keyword parameters that will be printed out
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`.
+              When successful, the message field contains a string with all the attributes.
+    """
+    tmp_logger = LogWrapper(_logger, "get_attributes")
+    tmp_logger.debug("Start")
+
+    # Add the parameters
+    parameter_dictionary = {key: str(value) for key, value in sorted(kwargs.items())}
+    parameter_section = "===== param =====\n" + "\n".join(f"{key} = {value}" for key, value in parameter_dictionary.items())
+
+    # Add the environment variables
+    environment_dictionary = {key: str(req.subprocess_env[key]) for key in sorted(req.subprocess_env)}
+    environment_section = "\n====== env ======\n" + "\n".join(f"{key} : {value}" for key, value in environment_dictionary.items())
+
+    # Combine sections
+    text_representation = parameter_section + "\n" + environment_section + "\n"
+
+    # Combine data in dictionary form
+    combined_data = {"parameters": parameter_dictionary, "environment": environment_dictionary}
+    tmp_logger.debug("Done")
+    return generate_response(True, text_representation, combined_data)
+
+
+@request_validation(_logger, secure=True, request_method="GET")
+def get_voms_attributes(req: PandaRequest) -> Dict:
+    """
+    Get VOMS attributes
+
+    Gets the VOMS attributes (i.e. the ones starting with GRST) in sorted order. Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/system/get_voms_attributes
+
+    Args:
+        req(PandaRequest): internally generated request object containing the env variables
+
+    Returns:
+        dict: The system response with text representation and dictionary of attributes.
+              Example: `{"success": True, "message": "<formatted string>", "data": {"key": "value"}}`
+    """
+    tmp_logger = LogWrapper(_logger, "get_voms_attributes")
+    tmp_logger.debug("Start")
+
+    # Iterate over all the environment variables, keep only the ones related to GRST credentials (GRST: Grid Security Technology)
+    attributes = {}
+    for tmp_key, tmp_value in req.subprocess_env.items():
+        if tmp_key.startswith("GRST_CRED_"):
+            attributes[tmp_key] = tmp_value
+
+    # Create a sorted text representation
+    text_representation = "\n".join(f"{key} : {value}" for key, value in sorted(attributes.items()))
+
+    tmp_logger.debug("Done")
+    return generate_response(True, text_representation, attributes)
+
+
+@request_validation(_logger, secure=True, request_method="GET")
+def get_user_attributes(req: PandaRequest) -> Dict:
+    """
+    Get user attributes
+
+    Gets user attributes as seen by PanDA for debug purposes. Requires a secure connection.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/system/get_user_attributes
+
+    Args:
+        req(PandaRequest): internally generated request object containing the env variables
+
+    Returns:
+        dict: The system response `{"success": success, "message": message, "data": data}`.
+              When successful, the message field contains a string with the user attributes,
+              and the data field contains the dictionary representation.
+    """
+    tmp_logger = LogWrapper(_logger, "get_user_attributes")
+    tmp_logger.debug("Start")
+
+    # Raw user DN
+    user_raw = req.subprocess_env["SSL_CLIENT_S_DN"]
+
+    # Bare user DN
+    user_bare = get_dn(req)
+
+    # Clean user DN
+    user_clean = clean_user_id(user_raw)
+
+    # FQANs
+    fqans = get_fqan(req)
+
+    # Email address
+    email_address = get_email_address(user_clean, tmp_logger)
+
+    # Has production role
+    is_production_user = has_production_role(req)
+
+    # Production working groups
+    production_working_groups = extract_production_working_groups(fqans)
+
+    # Primary production working group
+    primary_working_group = extract_primary_production_working_group(fqans)
+
+    # Combine sections
+    text_representation = (
+        f"User DN (raw): {user_raw}\nUser DN (bare): {user_bare}\nUser DN (clean): {user_clean}\n"
+        f"Email address: {email_address}\n"
+        f"FQANs: {fqans}\nProduction user: {is_production_user}\n"
+        f"Production working groups: {production_working_groups}\nPrimary production working group: {primary_working_group}\n"
+    )
+    dictionary_representation = {
+        "user_dn_raw": user_raw,
+        "user_dn_clean": user_clean,
+        "fqans": fqans,
+        "is_production_user": is_production_user,
+        "production_working_groups": production_working_groups,
+        "primary_working_group": primary_working_group,
+    }
+
+    tmp_logger.debug(text_representation)
+
+    tmp_logger.debug("Done")
+    return generate_response(True, text_representation, dictionary_representation)
+
+
+@request_validation(_logger, secure=False, request_method="GET")
+def is_alive(req: PandaRequest) -> Dict:
+    """
+    Is alive
+
+    Check if the server is alive. Basic function for the health check used in SLS monitoring.
+
+    API details:
+        HTTP Method: GET
+        Path: /v1/system/is_alive
+
+    Args:
+        req(PandaRequest): internally generated request object containing the env variables
+
+    Returns:
+        dict: The system response with the name of the endpoint
+              Example: `{"success": True}`
+    """
+    tmp_logger = LogWrapper(_logger, "is_alive")
+    tmp_logger.debug("Start")
+    tmp_logger.debug("Done")
+
+    return generate_response(True)