levelapp 0.1.15__py3-none-any.whl

levelapp/simulator/simulator.py

@@ -0,0 +1,531 @@
+"""
+'simulators/service.py': Service layer to manage conversation simulation and evaluation.
+"""
+import time
+import asyncio
+
+from datetime import datetime
+from collections import defaultdict
+from typing import Dict, Any, List
+
+
+from levelapp.core.base import BaseProcess, BaseEvaluator
+from levelapp.endpoint.client import EndpointConfig
+from levelapp.endpoint.manager import EndpointConfigManager
+
+from levelapp.core.schemas import EvaluatorType
+from levelapp.simulator.schemas import (
+    InteractionEvaluationResults,
+    ScriptsBatch,
+    ConversationScript,
+    SimulationResults, SingleInteractionResults, SingleAttemptResults, AllAttemptsResults
+)
+from levelapp.simulator.utils import (
+    calculate_average_scores,
+    summarize_verdicts,
+)
+from levelapp.aspects import logger
+
+
+class ConversationSimulator(BaseProcess):
+    """Conversation simulator component."""
+
+    def __init__(
+        self,
+        endpoint_config: EndpointConfig | None = None,
+        evaluators: Dict[EvaluatorType, BaseEvaluator] | None = None,
+        providers: List[str] | None = None,
+
+    ):
+        """
+        Initialize the ConversationSimulator.
+
+        Args:
+            endpoint_config (EndpointConfig): Endpoint configuration.
+            evaluators (EvaluationService): Service for evaluating interactions.
+            endpoint_config (EndpointConfig): Configuration object for VLA.
+        """
+        self._CLASS_NAME = self.__class__.__name__
+
+        self.endpoint_config = endpoint_config
+        self.evaluators = evaluators
+        self.providers = providers
+
+        self.endpoint_cm = EndpointConfigManager()
+
+        self.test_batch: ScriptsBatch | None = None
+
+    def setup(
+            self,
+            endpoint_config: EndpointConfig,
+            evaluators: Dict[EvaluatorType, BaseEvaluator],
+            providers: List[str],
+    ) -> None:
+        """
+        Initialize the ConversationSimulator.
+
+        Args:
+            endpoint_config (EndpointConfig): Configuration object for user endpoint API.
+            evaluators (Dict[str, BaseEvaluator]): List of evaluator objects for evaluating interactions.
+            providers (List[str]): List of LLM provider names.
+
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.setup.__name__}]"
+        logger.info(f"{_LOG} Setting up the Conversation Simulator..")
+
+        if not self.endpoint_cm:
+            self.endpoint_cm = EndpointConfigManager()
+
+        self.endpoint_config = endpoint_config
+        self.endpoint_cm.set_endpoints(endpoints_config=[endpoint_config])
+
+        self.evaluators = evaluators
+        self.providers = providers
+
+        if not self.providers:
+            logger.warning(f"{_LOG} No LLM providers were provided. The Judge Evaluation process will not be executed.")
+
+    def get_evaluator(self, name: EvaluatorType) -> BaseEvaluator:
+        """
+        Retrieve an evaluator by name.
+
+        Args:
+            name (EvaluatorType): Name of evaluator.
+
+        Returns:
+            An evaluator object.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.get_evaluator.__name__}]"
+
+        if name not in self.evaluators:
+            raise KeyError(f"{_LOG} Evaluator {name} not registered.")
+
+        return self.evaluators[name]
+
+    async def run(
+        self,
+        test_batch: ScriptsBatch,
+        attempts: int = 1,
+        batch_size: int = 4
+    ) -> Any:
+        """
+        Run a batch test for the given batch name and details.
+
+        Args:
+            test_batch (ScriptsBatch): Scenario batch object.
+            attempts (int): Number of attempts to run the simulation.
+            batch_size (int): Maximum number of concurrent processes to run the simulation.
+
+        Returns:
+            Dict[str, Any]: The results of the batch test.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.run.__name__}]"
+        logger.info(f"{_LOG} Starting batch test [attempts:{attempts}][batch-size:{batch_size}].")
+
+        started_at = datetime.now()
+
+        self.test_batch = test_batch
+        conversation_results = await self.simulate_conversation(attempts=attempts, max_concurrency=batch_size)
+
+        finished_at = datetime.now()
+
+        script_results: List[AllAttemptsResults] = conversation_results.get("script_results", [])
+
+        batch_verdicts: Dict[str, List[str]] = defaultdict(list)
+
+        for script in script_results:
+            for attempt in script.attempts:
+                for judge, verdicts in attempt.evaluation_verdicts.items():
+                    batch_verdicts[judge].extend(verdicts)
+
+        verdict_summaries: Dict[str, List[str]] = {
+            judge: summarize_verdicts(
+                verdicts=verdicts,
+                judge=judge,
+            )
+            for judge, verdicts in batch_verdicts.items()
+        }
+
+        results = SimulationResults(
+            started_at=started_at,
+            finished_at=finished_at,
+            evaluation_summary=verdict_summaries,
+            average_scores=conversation_results.get("average_scores", {}),
+            script_results=script_results
+        )
+
+        return results.model_dump_json(indent=2)
+
+    async def simulate_conversation(
+            self,
+            attempts: int = 1,
+            max_concurrency: int = 4,
+    ) -> Dict[str, Any]:
+        """
+        Simulate conversations for all scenarios in the batch.
+
+        Args:
+            attempts (int): Number of attempts to run the simulation.
+            max_concurrency (int): Maximum number of concurrent conversations.
+
+        Returns:
+            Dict[str, Any]: The results of the conversation simulation.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.simulate_conversation.__name__}]"
+        logger.info(f"{_LOG} starting conversation simulation..")
+
+        semaphore = asyncio.Semaphore(value=max_concurrency)
+
+        async def run_script(script: ConversationScript) -> AllAttemptsResults:
+            async with semaphore:
+                return await self.simulate_single_scenario(script=script, attempts=attempts)
+
+        scripts_tasks = [run_script(script=script) for script in self.test_batch.scripts]
+        script_results: List[AllAttemptsResults] = await asyncio.gather(*scripts_tasks)
+
+        aggregate_scores: Dict[str, List[float]] = defaultdict(list)
+
+        for result in script_results:
+            for metric, value in result.average_scores.items():
+                if isinstance(value, (int, float)):
+                    aggregate_scores[metric].append(value)
+
+        overall_average_scores = calculate_average_scores(aggregate_scores)
+
+        return {"script_results": script_results, "average_scores": overall_average_scores}
+
+    async def simulate_single_scenario(
+        self,
+        script: ConversationScript,
+        attempts: int = 1
+    ) -> AllAttemptsResults:
+        """
+        Simulate a single scenario with the given number of attempts, concurrently.
+
+        Args:
+            script (SimulationScenario): The scenario to simulate.
+            attempts (int): Number of attempts to run the simulation.
+
+        Returns:
+            AllAttemptsResults: The results of the scenario simulation attempts.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.simulate_single_scenario.__name__}]"
+
+        logger.info(f"{_LOG} Starting simulation for script: {script.id}")
+
+        async def simulate_attempt(attempt_number: int) -> SingleAttemptResults:
+            from uuid import uuid4
+            attempt_id: str = str(uuid4())
+
+            logger.info(f"{_LOG} Running attempt: {attempt_number + 1}/{attempts}\n---")
+            start_time = time.time()
+
+            interaction_results = await self.simulate_interactions(
+                script=script,
+                attempt_id=attempt_id,
+            )
+
+            collected_scores: Dict[str, List[Any]] = defaultdict(list)
+            collected_verdicts: Dict[str, List[Any]] = defaultdict(list)
+
+            for interaction in interaction_results:
+                if not interaction.evaluation_results:
+                    continue
+
+                eval_results = interaction.evaluation_results
+
+                # Judge scores & verdicts
+                for provider, judge_result in eval_results.judge_evaluations.items():
+                    collected_scores[provider].append(judge_result.score)
+                    collected_verdicts[provider].append(judge_result.justification)
+
+                # Metadata scores
+                if eval_results.metadata_evaluation:
+                    for _, score in eval_results.metadata_evaluation.items():
+                        collected_scores["metadata"].append(score)
+
+                # Guardrail
+                if eval_results.guardrail_flag is not None:
+                    collected_scores["guardrail"].append(eval_results.guardrail_flag)
+
+            elapsed_time = time.time() - start_time
+            collected_scores["processing_time"].append(elapsed_time)
+
+            average_scores = calculate_average_scores(collected_scores)
+
+            logger.info(f"{_LOG} Attempt {attempt_number + 1} completed in {elapsed_time:.2f}s\n---")
+
+            return SingleAttemptResults(
+                attempt_nbr=attempt_number + 1,
+                attempt_id=attempt_id,
+                script_id=str(script.id),
+                total_duration=elapsed_time,
+                interaction_results=interaction_results,
+                evaluation_verdicts=collected_verdicts,
+                average_scores=average_scores,
+            )
+
+        attempt_tasks = [simulate_attempt(i) for i in range(attempts)]
+        all_attempts: List[SingleAttemptResults] = await asyncio.gather(*attempt_tasks, return_exceptions=False)
+
+        scenario_scores: Dict[str, List[float]] = defaultdict(list)
+
+        for attempt in all_attempts:
+            for metric, value in attempt.average_scores.items():
+                if isinstance(value, (int, float)):
+                    scenario_scores[metric].append(value)
+
+        scenario_average_scores = calculate_average_scores(scenario_scores)
+
+        return AllAttemptsResults(
+            script_id=str(script.id),
+            attempts=all_attempts,
+            average_scores=scenario_average_scores,
+        )
+
+    async def simulate_interactions(
+        self,
+        script: ConversationScript,
+        attempt_id: str,
+    ) -> List[SingleInteractionResults]:
+        """
+        Simulate inbound interactions for a scenario.
+
+        Args:
+            script (ConversationScript): The script to simulate.
+            attempt_id (str): The id of the attempt.
+
+        Returns:
+            List[SingleInteractionResults]: The results of the inbound interactions simulation.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.simulate_interactions.__name__}]"
+
+        logger.info(f"{_LOG} Starting interactions simulation [ConvId:{attempt_id}]..")
+        start_time = time.time()
+
+        results = []
+        contextual_mode: bool = script.variable_request_schema
+        logger.info(f"{_LOG} Contextual Mode ON: {contextual_mode}")
+        interactions = script.interactions
+
+        for idx, interaction in enumerate(interactions):
+            request_payload = interaction.request_payload.copy()
+            if contextual_mode:
+                from levelapp.simulator.utils import set_by_path
+
+                if script.uuid_field:
+                    request_payload[script.uuid_field] = attempt_id
+
+                user_message = interaction.user_message
+                set_by_path(
+                    obj=request_payload,
+                    path=interaction.user_message_path,
+                    value=user_message,
+                )
+                logger.info(f"{_LOG} Request payload (Preloaded Request Schema):\n{request_payload}\n---")
+
+            else:
+                user_message = interaction.user_message
+                request_payload.update({"user_message": user_message})
+                logger.info(f"{_LOG} Request payload (Configured Request Schema):\n{request_payload}\n---")
+
+            mappings = self.endpoint_config.response_mapping
+
+            client_response = await self.endpoint_cm.send_request(
+                endpoint_config=self.endpoint_config,
+                context=request_payload,
+                contextual_mode=contextual_mode
+            )
+
+            reference_reply = interaction.reference_reply
+            reference_metadata = interaction.reference_metadata
+            reference_guardrail_flag: bool = interaction.guardrail_flag
+
+            if not client_response.response or client_response.response.status_code != 200:
+                logger.error(
+                    f"{_LOG} Interaction request failed [{client_response.error}]:\n{client_response.response}\n---"
+                )
+                output: SingleInteractionResults = SingleInteractionResults(
+                    conversation_id=attempt_id,
+                    user_message=user_message,
+                    reference_reply=reference_reply,
+                    reference_metadata=reference_metadata,
+                    errors={"error": str(client_response.error), "context": str(client_response.response)}
+                )
+                results.append(output)
+                continue
+
+            logger.info(
+                f"{_LOG} Response [{client_response.response.status_code}]:\n{client_response.response.text}\n---"
+            )
+
+            interaction_details = self.endpoint_cm.extract_response_data(
+                response=client_response.response,
+                mappings=mappings,
+            )
+
+            logger.info(f"{_LOG} Interaction details <ConvID:{attempt_id}>:\n{interaction_details}\n---")
+
+            generated_reply = interaction_details.get("agent_reply", "")
+            generated_metadata = interaction_details.get("metadata", {})
+            extracted_guardrail_flag = interaction_details.get("guardrail_flag", False)
+
+            logger.info(f"{_LOG} Generated reply <ConvID:{attempt_id}>:\n{generated_reply}\n---")
+
+            evaluation_results = await self.evaluate_interaction(
+                user_input=user_message,
+                generated_reply=generated_reply,
+                reference_reply=reference_reply,
+                generated_metadata=generated_metadata,
+                reference_metadata=reference_metadata,
+                generated_guardrail=extracted_guardrail_flag,
+                reference_guardrail=reference_guardrail_flag,
+            )
+
+            elapsed_time = time.time() - start_time
+            logger.info(f"{_LOG} Interaction simulation complete in {elapsed_time:.2f} seconds.\n---")
+
+            output: SingleInteractionResults = SingleInteractionResults(
+                conversation_id=attempt_id,
+                user_message=user_message,
+                generated_reply=generated_reply,
+                reference_reply=reference_reply,
+                generated_metadata=generated_metadata,
+                reference_metadata=reference_metadata,
+                guardrail_details=extracted_guardrail_flag,
+                evaluation_results=evaluation_results,
+                response_content=client_response.response.json(),
+            )
+
+            results.append(output)
+
+        return results
+
+    async def evaluate_interaction(
+        self,
+        user_input: str,
+        generated_reply: str,
+        reference_reply: str,
+        generated_metadata: Dict[str, Any],
+        reference_metadata: Dict[str, Any],
+        generated_guardrail: bool,
+        reference_guardrail: bool,
+    ) -> InteractionEvaluationResults:
+        """
+        Evaluate an interaction using OpenAI and Ionos evaluation services.
+
+        Args:
+            user_input (str): user input to evaluate.
+            generated_reply (str): The generated agent reply.
+            reference_reply (str): The reference agent reply.
+            generated_metadata (Dict[str, Any]): The generated metadata.
+            reference_metadata (Dict[str, Any]): The reference metadata.
+            generated_guardrail (bool): generated handoff/guardrail flag.
+            reference_guardrail (bool): reference handoff/guardrail flag.
+
+        Returns:
+            InteractionEvaluationResults: The evaluation results.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][{self.evaluate_interaction.__name__}]"
+
+        judge_evaluator: BaseEvaluator | None = self.evaluators.get(EvaluatorType.JUDGE, None)
+        metadata_evaluator: BaseEvaluator | None = self.evaluators.get(EvaluatorType.REFERENCE, None)
+
+        evaluation_results = InteractionEvaluationResults()
+
+        if judge_evaluator and self.providers:
+            await self._judge_evaluation(
+                user_input=user_input,
+                generated_reply=generated_reply,
+                reference_reply=reference_reply,
+                providers=self.providers,
+                judge_evaluator=judge_evaluator,
+                evaluation_results=evaluation_results,
+            )
+        else:
+            logger.info(f"{_LOG} Judge evaluation skipped (no evaluator or no providers).")
+
+        if metadata_evaluator and reference_metadata:
+            self._metadata_evaluation(
+                metadata_evaluator=metadata_evaluator,
+                generated_metadata=generated_metadata,
+                reference_metadata=reference_metadata,
+                evaluation_results=evaluation_results,
+            )
+        else:
+            logger.info(f"{_LOG} Metadata evaluation skipped (no evaluator or no reference metadata).")
+
+        evaluation_results.guardrail_flag = 1 if generated_guardrail == reference_guardrail else 0
+
+        return evaluation_results
+
+    async def _judge_evaluation(
+            self,
+            user_input: str,
+            generated_reply: str,
+            reference_reply: str,
+            providers: List[str],
+            judge_evaluator: BaseEvaluator,
+            evaluation_results: InteractionEvaluationResults,
+    ) -> None:
+        """
+        Run LLM-as-a-judge evaluation using multiple providers (async).
+
+        Args:
+            user_input (str): The user input message.
+            generated_reply (str): The generated agent reply.
+            reference_reply (str): The reference agent reply.
+            providers (List[str]): List of judge provider names.
+            judge_evaluator (BaseEvaluator): Evaluator instance.
+            evaluation_results (InteractionEvaluationResults): Results container (Pydantic model).
+
+        Returns:
+            None
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][judge_evaluation]"
+
+        tasks = {
+            provider: judge_evaluator.async_evaluate(
+                generated_data=generated_reply,
+                reference_data=reference_reply,
+                user_input=user_input,
+                provider=provider,
+            )
+            for provider in providers
+        }
+
+        results = await asyncio.gather(*tasks.values(), return_exceptions=True)
+
+        for provider, result in zip(tasks.keys(), results):
+            if isinstance(result, Exception):
+                logger.error(f"{_LOG} Provider '{provider}' failed to perform Judge Evaluation.")
+                evaluation_results.errors = {"provider": provider, "content": str(result)}
+            else:
+                evaluation_results.judge_evaluations[provider] = result
+
+    def _metadata_evaluation(
+            self,
+            metadata_evaluator: BaseEvaluator,
+            generated_metadata: Dict[str, Any],
+            reference_metadata: Dict[str, Any],
+            evaluation_results: InteractionEvaluationResults,
+    ) -> None:
+        """
+        Run metadata evaluation using the provided evaluator.
+
+        Args:
+            metadata_evaluator (BaseEvaluator): Evaluator for metadata comparison.
+            generated_metadata (Dict[str, Any]): The generated metadata.
+            reference_metadata (Dict[str, Any]): The reference metadata.
+            evaluation_results (InteractionEvaluationResults): Results container.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][metadata_evaluation]"
+
+        try:
+            evaluation_results.metadata_evaluation = metadata_evaluator.evaluate(
+                generated_data=generated_metadata,
+                reference_data=reference_metadata,
+            )
+        except Exception as e:
+            logger.error(f"{_LOG} Metadata evaluation failed:\n{e}", exc_info=e)
+            evaluation_results.errors = {"errors": e}

levelapp/simulator/utils.py

@@ -0,0 +1,134 @@
+"""
+'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
+"""
+import httpx
+
+from typing import Any, Dict, List, Union
+
+
+from levelapp.clients import ClientRegistry
+from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
+from levelapp.aspects import MonitoringAspect, MetricType, logger
+
+
+def set_by_path(obj: Dict, path: str, value: Any) -> None:
+    """
+    Sets a value in a nested dictionary using JSON path-like notation.
+
+    Args:
+        obj (dict): Dictionary to modify.
+        path (str): Path (e.g., "a.b[0].c") indicating where to set the value.
+        value (Any): Value to assign at the specified path.
+
+    Returns:
+        None
+    """
+    parts = path.split(".")
+    current = obj
+
+    for i, part in enumerate(parts):
+        is_last = i == len(parts) - 1
+
+        try:
+            # Handle list index access, e.g., key[0] or [1]
+            if '[' in part and ']' in part:
+                key, idx = part.split('[')
+                idx = int(idx.rstrip(']'))
+
+                # If we have a key before the list
+                if key:
+                    if key not in current or not isinstance(current[key], list):
+                        current[key] = []
+                    while len(current[key]) <= idx:
+                        current[key].append({})
+                    target = current[key]
+                else:
+                    if not isinstance(current, list):
+                        print("[set_by_path][WARNING] Expected a list at this level.")
+                        return
+                    while len(current) <= idx:
+                        current.append({})
+                    target = current
+
+                if is_last:
+                    target[idx] = value
+                else:
+                    if not isinstance(target[idx], dict):
+                        target[idx] = {}
+                    current = target[idx]
+
+            else:
+                # Regular dictionary key
+                if is_last:
+                    current[part] = value
+                else:
+                    if part not in current or not isinstance(current[part], dict):
+                        current[part] = {}
+                    current = current[part]
+
+        except (KeyError, IndexError, TypeError, AttributeError) as e:
+            print(f"[set_by_path][ERROR] Error type <{e.__class__.__name__}> : {e.args[0]}")
+            return
+
+
+@MonitoringAspect.monitor(
+    name="average_calc",
+    category=MetricType.SCORING,
+    cached=True,
+    maxsize=1000
+)
+def calculate_average_scores(scores: Dict[str, Union[List[float], float]]) -> Dict[str, float]:
+    """
+    Helper function that calculates the average scores for a dictionary of score lists.
+
+    Args:
+        scores (Dict[str, List[float]]): A dictionary where keys are identifiers and values are lists of scores.
+
+    Returns:
+        Dict[str, float]: A dictionary with average scores rounded to three decimal places.
+    """
+    result: Dict[str, float] = {}
+    for field, value in scores.items():
+        if isinstance(value, (int, float)):
+            result[field] = value
+        elif isinstance(value, list):
+            result[field] = round((sum(value) / len(value)), 3) if value else 0.0
+        else:
+            raise TypeError(f"[calculate_average_scores] Unexpected type '{type(value)}' for field '{field}")
+
+    return result
+
+
+@MonitoringAspect.monitor(name="summarization", category=MetricType.API_CALL)
+def summarize_verdicts(
+        verdicts: List[str],
+        judge: str,
+        max_bullets: int = 5
+) -> List[str]:
+    client_registry = ClientRegistry()
+    client = client_registry.get(provider=judge)
+
+    try:
+        verdicts = chr(10).join(verdicts)
+        prompt = SUMMARIZATION_PROMPT_TEMPLATE.format(max_bullets=max_bullets, judge=judge, verdicts=verdicts)
+        response = client.call(message=prompt)
+        parsed = client.parse_response(response=response)
+        striped = parsed.get("output", "").strip("")
+        bullet_points = [point.strip() for point in striped.split("- ") if point.strip()]
+
+        return bullet_points[:max_bullets]
+
+    except Exception as e:
+        logger.error(f"[summarize_justifications] Error during summarization: {str(e)}", exc_info=True)
+        return []
+
+
+# if __name__ == '__main__':
+#     template = {'generated_reply': '${agent_reply}', 'generated_metadata': '${generated_metadata}'}
+#     response_dict = {
+#         'agent_reply': "I'd be happy to help you book something for 10 AM.",
+#         'generated_metadata': {'appointment_type': 'Cardiology', 'date': 'next Monday', 'time': '10 AM'}
+#     }
+#
+#     result = extract_interaction_details(response_dict, template)
+#     print(f"result: {result.model_dump()}")

levelapp/visualization/__init__.py

@@ -0,0 +1,7 @@
+"""levelapp/visualization: Visualization module for evaluation results."""
+
+from .charts import ChartGenerator
+from .dashboard import DashboardGenerator
+from .exporter import ResultsExporter
+
+__all__ = ["ChartGenerator", "DashboardGenerator", "ResultsExporter"]