pygeai 0.1.6__py3-none-any.whl → 0.6.0b15__py3-none-any.whl

pygeai/evaluation/plan/clients.py

@@ -0,0 +1,302 @@
+import json
+
+from pygeai.evaluation.clients import EvaluationClient
+from pygeai.evaluation.plan.endpoints import LIST_EVALUATION_PLANS, CREATE_EVALUATION_PLAN, GET_EVALUATION_PLAN, \
+    UPDATE_EVALUATION_PLAN, DELETE_EVALUATION_PLAN, LIST_EVALUATION_PLAN_SYSTEM_METRICS, \
+    ADD_EVALUATION_PLAN_SYSTEM_METRIC, GET_EVALUATION_PLAN_SYSTEM_METRIC, UPDATE_EVALUATION_PLAN_SYSTEM_METRIC, \
+    DELETE_EVALUATION_PLAN_SYSTEM_METRIC, LIST_SYSTEM_METRICS, GET_SYSTEM_METRIC, EXECUTE_EVALUATION_PLAN
+from pygeai.core.utils.validators import validate_status_code
+from pygeai.core.utils.parsers import parse_json_response
+
+
+class EvaluationPlanClient(EvaluationClient):
+
+    def list_evaluation_plans(self) -> dict:
+        """
+        Retrieves a list of all evaluation plans.
+
+        :return: dict - API response containing a list of evaluation plans.
+        """
+        response = self.api_service.get(
+            endpoint=LIST_EVALUATION_PLANS
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def create_evaluation_plan(
+            self,
+            name: str,
+            type: str,
+            assistant_id: str = None,
+            assistant_name: str = None,
+            assistant_revision: str = None,
+            profile_name: str = None,
+            dataset_id: str = None,
+            system_metrics: list = None,
+    ) -> dict:
+        """
+        Creates a new evaluation plan.
+
+        :param name: str - Name of the evaluation plan.
+        :param type: str - Type of assistant (e.g., "TextPromptAssistant", "RAG Assistant").
+        :param assistant_id: str (optional) - ID of the assistant (required for "TextPromptAssistant").
+        :param assistant_name: str (optional) - Name of the assistant (required for "TextPromptAssistant").
+        :param assistant_revision: str (optional) - Revision of the assistant (required for "TextPromptAssistant").
+        :param profile_name: str (optional) - Name of the RAG profile (required for "RAG Assistant").
+        :param dataset_id: str (optional) - ID of the dataset.
+        :param system_metrics: list (optional) - Array of system metrics, each containing:
+            - systemMetricId: str - ID of the system metric.
+            - systemMetricWeight: float - Weight of the system metric (between 0 and 1).
+
+        :return: dict - API response with the created evaluation plan.
+        """
+        data = {
+            "evaluationPlanName": name,
+            "evaluationPlanType": type,
+            "systemMetrics": system_metrics,
+        }
+        if assistant_id is not None:
+            data["evaluationPlanAssistantId"] = assistant_id
+        if assistant_name is not None:
+            data["evaluationPlanAssistantName"] = assistant_name
+        if assistant_revision is not None:
+            data["evaluationPlanAssistantRevision"] = assistant_revision
+        if profile_name is not None:
+            data["evaluationPlanProfileName"] = profile_name
+        if dataset_id is not None:
+            data["dataSetId"] = dataset_id
+
+        response = self.api_service.post(
+            endpoint=CREATE_EVALUATION_PLAN,
+            data=data
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def get_evaluation_plan(self, evaluation_plan_id: str) -> dict:
+        """
+        Retrieves a specific evaluation plan by ID.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+
+        :return: dict - The evaluation plan metadata as a dictionary.
+        """
+        endpoint = GET_EVALUATION_PLAN.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.get(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def update_evaluation_plan(
+            self,
+            evaluation_plan_id: str,
+            name: str = None,
+            type: str = None,
+            assistant_id: str = None,
+            assistant_name: str = None,
+            assistant_revision: str = None,
+            profile_name: str = None,
+            dataset_id: str = None,
+            system_metrics: list = None,
+    ) -> dict:
+        """
+        Updates an existing evaluation plan by ID.
+
+        :param evaluation_plan_id: str - The unique identifier of the evaluation plan to update.
+        :param name: str, optional - The new name of the evaluation plan.
+        :param type: str, optional - The type of the evaluation plan.
+        :param assistant_id: str, optional - The ID of the assistant associated with the evaluation plan.
+        :param assistant_name: str, optional - The name of the assistant associated with the evaluation plan.
+        :param assistant_revision: str, optional - The revision identifier of the assistant.
+        :param profile_name: str, optional - The profile name associated with the evaluation plan.
+        :param dataset_id: str, optional - The ID of the dataset linked to the evaluation plan.
+        :param system_metrics: list, optional - A list of system metrics, each containing:
+            - systemMetricId (str): The metric identifier.
+            - systemMetricWeight (float): The weight of the metric (0 to 1).
+
+        :return: dict - API response containing the updated evaluation plan.
+        """
+        data = dict()
+
+        if name is not None:
+            data["evaluationPlanName"] = name
+        if type is not None:
+            data["evaluationPlanType"] = type
+        if system_metrics is not None:
+            data["systemMetrics"] = system_metrics
+        if assistant_id is not None:
+            data["evaluationPlanAssistantId"] = assistant_id
+        if assistant_name is not None:
+            data["evaluationPlanAssistantName"] = assistant_name
+        if assistant_revision is not None:
+            data["evaluationPlanAssistantRevision"] = assistant_revision
+        if profile_name is not None:
+            data["evaluationPlanProfileName"] = profile_name
+        if dataset_id is not None:
+            data["dataSetId"] = dataset_id
+
+        endpoint = UPDATE_EVALUATION_PLAN.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.put(
+            endpoint=endpoint,
+            data=data
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def delete_evaluation_plan(self, evaluation_plan_id: str) -> dict:
+        """
+        Deletes a specific evaluation plan by ID.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+
+        :return: dict - Response indicating the success or failure of the deletion.
+        """
+        endpoint = DELETE_EVALUATION_PLAN.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.delete(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def list_evaluation_plan_system_metrics(self, evaluation_plan_id: str) -> dict:
+        """
+        Retrieves system metrics associated with a specific evaluation plan.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+
+        :return: dict - List of system metrics for the evaluation plan.
+        """
+        endpoint = LIST_EVALUATION_PLAN_SYSTEM_METRICS.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.get(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def add_evaluation_plan_system_metric(
+            self,
+            evaluation_plan_id: str,
+            system_metric_id: str,
+            system_metric_weight: float
+    ) -> dict:
+        """
+        Adds a system metric to an existing evaluation plan.
+
+        :param evaluation_plan_id: str - The unique identifier of the evaluation plan.
+        :param system_metric_id: str - The unique identifier of the system metric to add.
+        :param system_metric_weight: float - The weight of the system metric (0 to 1).
+
+        :return: dict - API response containing the added system metric details.
+        """
+        data = {
+            "systemMetricId": system_metric_id,
+            "systemMetricWeight":system_metric_weight
+        }
+        endpoint = ADD_EVALUATION_PLAN_SYSTEM_METRIC.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.post(
+            endpoint=endpoint,
+            data=data
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def get_evaluation_plan_system_metric(self, evaluation_plan_id: str, system_metric_id: str) -> dict:
+        """
+        Retrieves a specific system metric from an evaluation plan.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+        :param system_metric_id: str - The ID of the system metric.
+
+        :return: dict - The system metric metadata.
+        """
+        endpoint = GET_EVALUATION_PLAN_SYSTEM_METRIC.format(evaluationPlanId=evaluation_plan_id, systemMetricId=system_metric_id)
+        response = self.api_service.get(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def update_evaluation_plan_system_metric(
+            self,
+            evaluation_plan_id: str,
+            system_metric_id: str,
+            system_metric_weight: float
+    ) -> dict:
+        """
+        Updates the weight of a system metric in an evaluation plan.
+
+        :param evaluation_plan_id: str - The unique identifier of the evaluation plan.
+        :param system_metric_id: str - The unique identifier of the system metric to update.
+        :param system_metric_weight: float - The new weight of the system metric (0 to 1).
+
+        :return: dict - API response containing the updated system metric details.
+        """
+        data = {
+            "systemMetricWeight": system_metric_weight
+        }
+        endpoint = UPDATE_EVALUATION_PLAN_SYSTEM_METRIC.format(evaluationPlanId=evaluation_plan_id, systemMetricId=system_metric_id)
+        response = self.api_service.put(
+            endpoint=endpoint,
+            data=data
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def delete_evaluation_plan_system_metric(self, evaluation_plan_id: str, system_metric_id: str) -> dict:
+        """
+        Deletes a specific system metric from an evaluation plan.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+        :param system_metric_id: str - The ID of the system metric.
+
+        :return: dict - Response indicating the success or failure of the deletion.
+        """
+        endpoint = DELETE_EVALUATION_PLAN_SYSTEM_METRIC.format(evaluationPlanId=evaluation_plan_id, systemMetricId=system_metric_id)
+        response = self.api_service.delete(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def list_system_metrics(self) -> dict:
+        """
+        Retrieves a list of all available system metrics.
+
+        :return: dict - List of all system metrics.
+        """
+        response = self.api_service.get(
+            endpoint=LIST_SYSTEM_METRICS
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def get_system_metric(self, system_metric_id: str) -> dict:
+        """
+        Retrieves a specific system metric by ID.
+
+        :param system_metric_id: str - The ID of the system metric.
+
+        :return: dict - The system metric metadata.
+        """
+        endpoint = GET_SYSTEM_METRIC.format(systemMetricId=system_metric_id)
+        response = self.api_service.get(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def execute_evaluation_plan(self, evaluation_plan_id: str) -> dict:
+        """
+        Executes a specific evaluation plan.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+
+        :return: dict - API response confirming the execution of the evaluation plan.
+        """
+        endpoint = EXECUTE_EVALUATION_PLAN.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.post(
+            endpoint=endpoint,
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")

pygeai/evaluation/plan/endpoints.py

@@ -0,0 +1,16 @@
+LIST_EVALUATION_PLANS = "evaluationPlanApi/evaluationPlans"  # GET -> Retrieves a list of all evaluation plans.
+CREATE_EVALUATION_PLAN = "evaluationPlanApi/evaluationPlan"  # POST -> Creates a new evaluation plan.
+GET_EVALUATION_PLAN = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}"  # GET -> Retrieves a specific evaluation plan by ID.
+UPDATE_EVALUATION_PLAN = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}"  # PUT -> Updates a specific evaluation plan by ID.
+DELETE_EVALUATION_PLAN = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}"  # DELETE -> Deletes a specific evaluation plan by ID.
+
+LIST_EVALUATION_PLAN_SYSTEM_METRICS = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}/evaluationPlanSystemMetrics"  # GET -> Retrieves system metrics associated with a specific evaluation plan.
+ADD_EVALUATION_PLAN_SYSTEM_METRIC = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}/evaluationPlanSystemMetric"  # POST -> Adds a system metric to a specific evaluation plan.
+GET_EVALUATION_PLAN_SYSTEM_METRIC = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}/evaluationPlanSystemMetric/{systemMetricId}"  # GET -> Retrieves a specific system metric from an evaluation plan.
+UPDATE_EVALUATION_PLAN_SYSTEM_METRIC = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}/evaluationPlanSystemMetric/{systemMetricId}"  # PUT -> Updates a specific system metric within an evaluation plan.
+DELETE_EVALUATION_PLAN_SYSTEM_METRIC = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}/evaluationPlanSystemMetric/{systemMetricId}"  # DELETE -> Deletes a specific system metric from an evaluation plan.
+
+LIST_SYSTEM_METRICS = "evaluationPlanApi/systemMetrics"  # GET -> Retrieves a list of all available system metrics.
+GET_SYSTEM_METRIC = "evaluationPlanApi/systemMetric/{systemMetricId}"  # GET -> Retrieves a specific system metric by ID.
+
+EXECUTE_EVALUATION_PLAN = "evaluationPlanApi/evaluationPlan/{evaluationPlanId}"  # POST -> Executes a specific evaluation plan.

pygeai/evaluation/result/clients.py

@@ -0,0 +1,70 @@
+import json
+
+from pygeai.evaluation.clients import EvaluationClient
+from pygeai.evaluation.result.endpoints import LIST_EVALUATION_RESULTS, GET_EVALUATION_RESULT
+from pygeai.core.utils.validators import validate_status_code
+from pygeai.core.utils.parsers import parse_json_response
+
+
+class EvaluationResultClient(EvaluationClient):
+    """
+    Client for interacting with the Evaluation Result API.
+    
+    This API is read-only and retrieves results from executed evaluation plans.
+    
+    .. warning::
+        The API documentation at https://docs.globant.ai/en/wiki?856,Evaluation+Result+API
+        contains several typos in field names. The actual API responses use these typo'd names:
+        
+        - evaluationResultAssitantRevision (missing 's' in Assistant)
+        - evaluationResultChunckCount (should be Chunk, not Chunck)
+        - evaluationResultChunckSize (should be Chunk, not Chunck)
+        - evaluationResultaMaxTokens (lowercase 'a' should be uppercase 'M', no 'a')
+        
+        Our implementation returns these fields as-is from the API.
+    """
+
+    def list_evaluation_results(self, evaluation_plan_id: str) -> dict:
+        """
+        Retrieves a list of evaluation results for a given evaluation plan ID.
+
+        :param evaluation_plan_id: str - The ID of the evaluation plan.
+
+        :return: dict - API response containing a list of evaluation results.
+        
+        .. note::
+            Response contains evaluation result objects with fields including:
+            dataSetId, evaluationPlanId, evaluationResultId, evaluationResultStatus,
+            evaluationResultCost, evaluationResultDuration, and others.
+            
+            See class documentation for field name typos in the API.
+        """
+        endpoint = LIST_EVALUATION_RESULTS.format(evaluationPlanId=evaluation_plan_id)
+        response = self.api_service.get(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")
+
+    def get_evaluation_result(self, evaluation_result_id: str) -> dict:
+        """
+        Retrieves a specific evaluation result by its ID.
+
+        :param evaluation_result_id: str - The ID of the evaluation result.
+
+        :return: dict - The evaluation result metadata as a dictionary, including row-level data.
+        
+        .. note::
+            Response includes all fields from list_evaluation_results plus a 'rows' array
+            containing detailed row-level evaluation data with fields like:
+            dataSetRowId, evaluationResultRowStatus, evaluationResultRowOutput,
+            evaluationResultRowCost, etc.
+            
+            See class documentation for field name typos in the API.
+        """
+        endpoint = GET_EVALUATION_RESULT.format(evaluationResultId=evaluation_result_id)
+        response = self.api_service.get(
+            endpoint=endpoint
+        )
+        validate_status_code(response)
+        return parse_json_response(response, "evaluation plan operation")

pygeai/evaluation/result/endpoints.py

@@ -0,0 +1,2 @@
+LIST_EVALUATION_RESULTS = "evaluationResultApi/evaluationResults/{evaluationPlanId}"  # GET -> Retrieves a list of evaluation results for a given evaluation plan ID.
+GET_EVALUATION_RESULT = "evaluationResultApi/evaluationResult/{evaluationResultId}"  # GET -> Retrieves a specific evaluation result by its ID.