ragbits-evaluate 0.0.30rc1__py3-none-any.whl → 1.4.0.dev202602030301__py3-none-any.whl

ragbits/evaluate/agent_simulation/simulation.py

@@ -2,47 +2,16 @@
 
 from __future__ import annotations
 
+import json
+import re
 from typing import TYPE_CHECKING
 
-from pydantic import BaseModel
-
+from ragbits.agents.tool import ToolCallResult
 from ragbits.core.llms import LiteLLM
-from ragbits.core.prompt import Prompt
 from ragbits.evaluate.agent_simulation.models import Personality, Scenario, Task, Turn
 
 if TYPE_CHECKING:
-    from ragbits.evaluate.agent_simulation.context import DataSnapshot
-
-
-class SimulatedUserPromptInput(BaseModel):
-    """Input for the simulated user prompt."""
-
-    scenario_name: str
-    task_context: str
-    personality_instruction: str
-    grounding_block: str
-    history_block: str
-    current_task: str
-
-
-class SimulatedUserPrompt(Prompt[SimulatedUserPromptInput, str]):
-    """Prompt for generating simulated user messages."""
-
-    system_prompt = """
-You are simulating a concise human user in a terminal chat.
-Scenario: {{ scenario_name }}
-{{ task_context }}{{ personality_instruction }}{{ grounding_block }}
-Given the assistant's last reply and the conversation so far,
-write ONLY the next user message to work on the current task. Be specific and brief.
-"""
-
-    user_prompt = """
-[CONVERSATION]
-{{ history_block }}
-
-[TASK]
-Write the next USER message now (follow task: {{ current_task }}):
-"""
+    from ragbits.evaluate.agent_simulation.context import DataSnapshot, DomainContext
 
 
 class SimulatedUser:
@@ -95,52 +64,177 @@ class SimulatedUser:
         if current_task is None:
             return "Thank you, all tasks are completed."
 
-        prompt = SimulatedUserPrompt(
-            SimulatedUserPromptInput(
-                scenario_name=self.scenario.name,
-                task_context=self._build_task_context(current_task),
-                personality_instruction=self._build_personality_instruction(),
-                grounding_block=self._build_grounding_block(),
-                history_block=self._build_history_block(history),
-                current_task=current_task.task,
-            )
-        )
-
-        response = await self.llm.generate(prompt)
-        return response.strip()
+        history_text = []
+        for t in history:
+            history_text.append(f"User: {t.user}\nAssistant: {t.assistant}")
+        history_block = "\n\n".join(history_text) if history_text else "(no prior messages)"
 
-    def _build_task_context(self, current_task: Task) -> str:
-        """Build the task context string."""
         task_context = f"Current task: {current_task.task}"
         if self.current_task_idx > 0:
             completed_tasks = ", ".join([t.task for t in self.scenario.tasks[: self.current_task_idx]])
             task_context += f"\nCompleted tasks: {completed_tasks}"
-        return task_context
 
-    def _build_personality_instruction(self) -> str:
-        """Build the personality instruction string."""
+        personality_instruction = ""
         if self.personality:
-            return f"\n\nPersonality: {self.personality.description}"
-        return ""
-
-    def _build_grounding_block(self) -> str:
-        """Build the data grounding block string."""
-        if not self.data_snapshot:
-            return ""
-
-        return f"""
-        <AVAILABLE_DATA comment="This section lists data available for agent, not listed entities may not exist.">
-            {self.data_snapshot.format_for_prompt()}
-        </AVAILABLE_DATA>
+            personality_instruction = f"\n\nPersonality: {self.personality.description}"
+
+        # Build data grounding block if snapshot is provided
+        grounding_block = ""
+        if self.data_snapshot:
+            grounding_block = (
+                "\n\n[AVAILABLE DATA]\n"
+                f"{self.data_snapshot.format_for_prompt()}\n\n"
+                "IMPORTANT: Only reference items that exist in the AVAILABLE DATA above. "
+                "Do not ask for entities that are not listed."
+            )
+
+        prompt = (
+            "[SYSTEM]\n"
+            "You are simulating a concise human user in a terminal chat. "
+            f"Scenario: {self.scenario.name}\n"
+            f"{task_context}{personality_instruction}{grounding_block}\n"
+            "Given the assistant's last reply and the conversation so far, "
+            "write ONLY the next user message to work on the current task. Be specific and brief.\n\n"
+            "[CONVERSATION]\n"
+            f"{history_block}\n\n"
+            "[TASK]\nWrite the next USER message now:"
+        )
+
+        response = await self.llm.generate(prompt=prompt)
+        return response.strip()
+
+
+class GoalChecker:
+    """A lightweight judge model that decides whether the current task has been achieved.
+
+    It inspects the conversation so far and checks if the task matches the expected result.
+    Supports optional domain context for accurate evaluation in specific domains.
+    """
+
+    def __init__(self, llm: LiteLLM, scenario: Scenario) -> None:
+        self.llm = llm
+        self.scenario = scenario
+
+    async def is_task_achieved(
+        self,
+        current_task: Task,
+        history: list[Turn],
+        context: DomainContext | None = None,
+    ) -> tuple[bool, str]:
+        """Check if the current task has been completed based on the conversation history.
+
+        Args:
+            current_task: The task to check completion for.
+            history: List of conversation turns so far.
+            context: Optional domain context for accurate evaluation (e.g., currency, locale).
+
+        Returns:
+            Tuple of (is_completed, reason).
         """
+        history_text = []
+        for t in history:
+            history_text.append(f"User: {t.user}\nAssistant: {t.assistant}")
+        history_block = "\n\n".join(history_text) if history_text else "(no prior messages)"
+
+        # Build context block if provided
+        context_block = ""
+        if context:
+            context_block = (
+                "\n[IMPORTANT CONTEXT]\n"
+                f"{context.format_for_prompt()}\n\n"
+                "When evaluating task completion, consider the domain context above "
+                f"and use {context.locale} locale conventions.\n\n"
+            )
+
+        prompt = (
+            "[SYSTEM]\n"
+            "You are a strict task-completion judge for a user-assistant conversation. "
+            "Decide if the assistant has fulfilled the current task.\n"
+            f"Current task: {current_task.task}\n"
+            f"Expected result: {current_task.expected_result}\n"
+            f"{context_block}"
+            "Respond with a concise JSON object ONLY, no extra text, with fields:\n"
+            '{"done": true|false, "reason": "short reason"}\n\n'
+            "[CONVERSATION]\n"
+            f"{history_block}\n\n"
+            "[TASK]\nReturn the JSON now:"
+        )
+
+        response = await self.llm.generate(prompt=prompt)
+        text = response.strip()
+        # Be robust to slight deviations by attempting a minimal parse
+        done = False
+        reason = ""
+
+        if not text:
+            return False, "Empty response from goal checker"
+
+        # Try to extract JSON from markdown code blocks if present
+        code_block_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+        if code_block_match:
+            text = code_block_match.group(1)
+
+        # Try to find JSON object in the text
+        json_match = re.search(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}", text, re.DOTALL)
+        if json_match:
+            text = json_match.group(0)
+
+        try:
+            data = json.loads(text)
+            done = bool(data.get("done", False))
+            reason = str(data.get("reason", "")).strip()
+        except json.JSONDecodeError:
+            # If JSON parsing fails, try to infer from response text
+            reason = f"Failed to parse JSON response: {text[:100]}"
+            # Heuristic: if response contains "done" or "completed" or "true", assume done
+            text_lower = text.lower()
+            if any(word in text_lower for word in ["done", "completed", "true", "yes", "success"]):
+                done = True
+            elif any(word in text_lower for word in ["not done", "incomplete", "false", "no", "failed"]):
+                done = False
+
+        return done, reason
+
+
+class ToolUsageChecker:
+    """A simple comparator that verifies whether the agent used the expected tools.
+
+    It checks if all expected tools from the task were called during the conversation turn.
+    """
+
+    def __init__(self, scenario: Scenario) -> None:
+        self.scenario = scenario
+
+    def check_tool_usage(self, current_task: Task, tool_calls: list[ToolCallResult]) -> tuple[bool, str]:  # noqa: PLR6301
+        """Check if the expected tools were used for the current task.
+
+        Args:
+            current_task: The current task being evaluated
+            tool_calls: List of tool calls made during this turn
+
+        Returns:
+            Tuple of (success: bool, reason: str)
+        """
+        if not current_task.expected_tools:
+            return True, "No expected tools specified"
+
+        if not tool_calls:
+            return False, "No tools were called, but tools were expected"
+
+        # Get the names of tools that were actually called
+        called_tool_names = [tc.name for tc in tool_calls]
+        expected_tool_names = current_task.expected_tools
+
+        # Check if all expected tools were used
+        missing_tools = [tool for tool in expected_tool_names if tool not in called_tool_names]
+
+        if missing_tools:
+            return (
+                False,
+                f"Expected tools not used: {', '.join(missing_tools)}. Tools called: {', '.join(called_tool_names)}",
+            )
 
-    @staticmethod
-    def _build_history_block(history: list[Turn]) -> str:
-        """Build the conversation history block string."""
-        if not history:
-            return "(no prior messages)"
-        history_text = [f"User: {t.user}\nAssistant: {t.assistant}" for t in history]
-        return "\n\n".join(history_text)
+        return True, f"All expected tools used: {', '.join(called_tool_names)}"
 
 
 def build_llm(model_name: str | None, default_model: str, api_key: str) -> LiteLLM:

ragbits/evaluate/metrics/question_answer.py

@@ -4,14 +4,6 @@ from asyncio import AbstractEventLoop
 from itertools import chain
 from typing import Generic, TypeVar
 
-from continuous_eval.llm_factory import LLMInterface
-from continuous_eval.metrics.base import LLMBasedMetric
-from continuous_eval.metrics.generation.text import (
-    LLMBasedAnswerCorrectness,
-    LLMBasedAnswerRelevance,
-    LLMBasedFaithfulness,
-    LLMBasedStyleConsistency,
-)
 from typing_extensions import Self
 
 from ragbits.agents.types import QuestionAnswerPromptOutputT
@@ -20,6 +12,31 @@ from ragbits.core.utils.helpers import batched
 from ragbits.evaluate.metrics.base import Metric
 from ragbits.evaluate.pipelines.question_answer import QuestionAnswerResult
 
+try:
+    from continuous_eval.llm_factory import LLMInterface
+    from continuous_eval.metrics.base import LLMBasedMetric
+    from continuous_eval.metrics.generation.text import (
+        LLMBasedAnswerCorrectness,
+        LLMBasedAnswerRelevance,
+        LLMBasedFaithfulness,
+        LLMBasedStyleConsistency,
+    )
+except ModuleNotFoundError:
+    from continuous_eval.llms.base import LLMInterface
+    from continuous_eval.metrics import Metric as LLMBasedMetric
+    from continuous_eval.metrics.generation.text import (
+        AnswerCorrectness as LLMBasedAnswerCorrectness,
+    )
+    from continuous_eval.metrics.generation.text import (
+        AnswerRelevance as LLMBasedAnswerRelevance,
+    )
+    from continuous_eval.metrics.generation.text import (
+        Faithfulness as LLMBasedFaithfulness,
+    )
+    from continuous_eval.metrics.generation.text import (
+        StyleConsistency as LLMBasedStyleConsistency,
+    )
+
 MetricT = TypeVar("MetricT", bound=LLMBasedMetric)
 
 

{ragbits_evaluate-0.0.30rc1.dist-info → ragbits_evaluate-1.4.0.dev202602030301.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragbits-evaluate
-Version: 0.0.30rc1
+Version: 1.4.0.dev202602030301
 Summary: Evaluation module for Ragbits components
 Project-URL: Homepage, https://github.com/deepsense-ai/ragbits
 Project-URL: Bug Reports, https://github.com/deepsense-ai/ragbits/issues
@@ -28,13 +28,9 @@ Requires-Dist: distilabel<2.0.0,>=1.5.0
 Requires-Dist: hydra-core<2.0.0,>=1.3.2
 Requires-Dist: neptune[optuna]<2.0.0,>=1.12.0
 Requires-Dist: optuna<5.0.0,>=4.0.0
-Requires-Dist: ragbits-core==0.0.30.rc1
-Provides-Extra: postgres
-Requires-Dist: ragbits-core[postgres]==1.4.0.dev202512110238; extra == 'postgres'
+Requires-Dist: ragbits-core==1.4.0.dev202602030301
 Provides-Extra: relari
 Requires-Dist: continuous-eval<1.0.0,>=0.3.12; extra == 'relari'
-Provides-Extra: sqlite
-Requires-Dist: ragbits-core[sqlite]==1.4.0.dev202512110238; extra == 'sqlite'
 Description-Content-Type: text/markdown
 
 # Ragbits Evaluate

{ragbits_evaluate-0.0.30rc1.dist-info → ragbits_evaluate-1.4.0.dev202602030301.dist-info}/RECORD

@@ -1,29 +1,22 @@
 ragbits/evaluate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ragbits/evaluate/api.py,sha256=OtM60L4WNAuPMvN1pE3vmWwq7ORD1YuYWI3ETsnpGk0,26223
-ragbits/evaluate/api_types.py,sha256=yDGkHf6ACeytK5NETYKKgaAW0aRaMmRoSib70ZvHvxM,8678
 ragbits/evaluate/cli.py,sha256=vP8l2DyNXpR6jQP83wXKP_RRnGjEXjKnTVBg9RPbDKo,4505
 ragbits/evaluate/config.py,sha256=2WSmbVxyQi893L2FSjRFQoXkWZp1GetcNmR2GCDe0tA,339
 ragbits/evaluate/evaluator.py,sha256=-VcO61r340lt6KWTjkl8DdHmU78WygBP7wfYLT2hu9k,8319
-ragbits/evaluate/execution_manager.py,sha256=elZ4iUhpvFazdLJPNyK5hWuPJmUFfFQNTwDHoy8uyus,15561
 ragbits/evaluate/optimizer.py,sha256=RqYgoiCIEhjXO0HEN6uwldblHyoPuT3qUdncuHPZgCg,8485
 ragbits/evaluate/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragbits/evaluate/utils.py,sha256=w-hbvKRHI9tEva9wKDTVla0Wm2eCHT2MxVkof27Sqfw,4831
-ragbits/evaluate/agent_simulation/__init__.py,sha256=CjEynRztU2lcG5U4pJzEuc2Aj2pZvXTj2fmXhUnOtSo,4370
-ragbits/evaluate/agent_simulation/checkers.py,sha256=oo-KzuPhq3Ur9B2ZIgs5MLEMpmgG0vdBD0SIba-I_-o,21452
+ragbits/evaluate/agent_simulation/__init__.py,sha256=Cf867LFsLLnDRwDO8rGFQFxvrLemR8qkSqkk5zYyxT0,2971
 ragbits/evaluate/agent_simulation/context.py,sha256=_gZcRpKtHOXoyR2LsMYWH2w9Wm7h2bozuSlNKsZC7ng,4196
-ragbits/evaluate/agent_simulation/conversation.py,sha256=CPaek0c1D5CUSx7-2aFe5hjUPwQaAIgCEkOPKdgegfI,25065
+ragbits/evaluate/agent_simulation/conversation.py,sha256=nQr3EHlaBu066rFdWXacyPNXtte6PEDoJnVXjymEifQ,14087
 ragbits/evaluate/agent_simulation/deepeval_evaluator.py,sha256=WDtrApfLS1W5-mB86Yg2xlFgPYVzLtvJV8wCiJs5y2w,3453
-ragbits/evaluate/agent_simulation/display.py,sha256=vGcu9TigrmZbfk5QgDz9LQIpkJIW4H12YvB0Mk_Vkhw,3777
-ragbits/evaluate/agent_simulation/logger.py,sha256=1FGSQjF7TW2UvYKL4BsMhmfeqVEQ5aeg-44rw7TG91w,6959
-ragbits/evaluate/agent_simulation/models.py,sha256=TufVe4j2Fd1h_2SQNUeR8uCrFEz9FlX3iMHf3Q-WM0s,7718
-ragbits/evaluate/agent_simulation/results.py,sha256=PhJO7lkkbOrQgZplEkNDwiWF42yN5advBaThWP4kIsA,9428
-ragbits/evaluate/agent_simulation/scenarios.py,sha256=irI64oN1BjHJ2nVfBuDbqE7d6ihnfSJyOxYIq7h9m4k,6583
-ragbits/evaluate/agent_simulation/simulation.py,sha256=1472N_pnlYtywtKYx_3zJtHoccbsOjZQYKLF4Wc3Bo4,5449
-ragbits/evaluate/agent_simulation/tracing.py,sha256=iuinYBXFWS-ookb5TmNatfDNdpTopNNTdbB3Qo9WWyM,8019
-ragbits/evaluate/agent_simulation/metrics/__init__.py,sha256=tIpNam1AEYRTBAe5IHPqbNtOPO5tf3qIv34fy4YHCoE,892
-ragbits/evaluate/agent_simulation/metrics/builtin.py,sha256=U3kFLeglQ64KPDOdUBx0VVhGOXoyCIwahxg-305GeXU,8086
-ragbits/evaluate/agent_simulation/metrics/collectors.py,sha256=eZSVV9KKxjbDzFyaIiUw1Vo-40W3y5tB3K_4-RFZ7Ww,5782
-ragbits/evaluate/agent_simulation/metrics/deepeval.py,sha256=iEoWF5y3_intueG4uJsC_AEGrCrb-V03vaAZGIiS00A,10956
+ragbits/evaluate/agent_simulation/logger.py,sha256=47lxIRM-DMvP3Cw4XGUVHSNxqjGOk9ozSgxCY6DOIIQ,6948
+ragbits/evaluate/agent_simulation/models.py,sha256=tOjEzK2Iv1_Nu-jT-soORgd15oqJ2zb8g3EXN7ZKn0c,719
+ragbits/evaluate/agent_simulation/results.py,sha256=EJcpJq-86NYPcY8KMNsmxH9aLXTWGoXZY5aig1fbdP4,6925
+ragbits/evaluate/agent_simulation/scenarios.py,sha256=FU0Z9eZUA7yMAZyAUJ3e5SxZR6XwWdZrbEsLz_GHbhI,4364
+ragbits/evaluate/agent_simulation/simulation.py,sha256=GiKutdl-RRIcL_p9sVxvGYCHGCDS1Rz3oQcwLt9rdi0,9633
+ragbits/evaluate/agent_simulation/metrics/__init__.py,sha256=3Gtix1JRAGyLe6JHQTGNr2gIrvEsErY7ys2UkPwdBVU,499
+ragbits/evaluate/agent_simulation/metrics/builtin.py,sha256=ZSK_cBpk1J_bnDes8mbjqw6cW4oml5BAGM7_EOgx6CE,7481
+ragbits/evaluate/agent_simulation/metrics/collectors.py,sha256=fTTYrYimyF0i6KsWagiU12XV0n7AeNaLYqy1n_faWdI,4796
 ragbits/evaluate/dataloaders/__init__.py,sha256=UFJFjmvi3GUQFsx6A5sYD01HH2f7TXcHRW2VNM1pmIA,83
 ragbits/evaluate/dataloaders/base.py,sha256=x8rEl5utNOziF_9urL0grkqoXwMgaDWYSM5akw3Kt9Y,3213
 ragbits/evaluate/dataloaders/document_search.py,sha256=c9Bc4ZtFEKAiG9B70JFiBZlZDkBSGNWFRKabF7PMTU0,2495
@@ -53,7 +46,7 @@ ragbits/evaluate/metrics/document_search.py,sha256=MfvMwEPenqiJdKYuW6WLvmtMch9ZV
 ragbits/evaluate/metrics/gaia.py,sha256=Q1oZPVAxRsQkyctJLE95fsGNewmDhzTJ5vjNoXvu10E,3086
 ragbits/evaluate/metrics/hotpot_qa.py,sha256=Tw4gKDbua60fbE7BbxKV08-yp0PbQKTHlnk87GULNe8,1776
 ragbits/evaluate/metrics/human_eval.py,sha256=ud4G-xaMi0f1tkzYb1V2uSgYDF-ymnKxiN6CdOWGZqU,4285
-ragbits/evaluate/metrics/question_answer.py,sha256=369lOoY76KY-wUxBKl0lSQlJSF0JhmPpehNQYeiWNHg,7072
+ragbits/evaluate/metrics/question_answer.py,sha256=4rbJ9z_eAyk_5mOeQGyaP_rqodS8MZZltwmPmDitb_4,7722
 ragbits/evaluate/pipelines/__init__.py,sha256=PZ2477OqOV622QMC-3iwW5ThC-nYRS9KBe_nlyas3Zs,1573
 ragbits/evaluate/pipelines/base.py,sha256=QV3fjPnbJjeCgcbt8yV1Ho3BamEUc3wSca3MAzaBlV0,1739
 ragbits/evaluate/pipelines/document_search.py,sha256=tgk-I21eshdBbWVsuNa1zWK_fWuDNXhhMCn1_Fdu_Ko,3840
@@ -61,10 +54,6 @@ ragbits/evaluate/pipelines/gaia.py,sha256=DkVAlNI-a9chQGPyOFtjrXWGoPSWogrSfSxxgx
 ragbits/evaluate/pipelines/hotpot_qa.py,sha256=eHDQ7e_Pa1YRWkc-7oxarYHnZKKvEv_Q8gCBBvo_iss,13629
 ragbits/evaluate/pipelines/human_eval.py,sha256=o2q3O3-OcdRBESwRFUKVU2dQt0TIMDXqxFitdKEw_fw,12406
 ragbits/evaluate/pipelines/question_answer.py,sha256=3CYVHDLnOy4z7kgYPMluiJ8POulHo-w3PEiqvqsF4Dc,2797
-ragbits/evaluate/stores/__init__.py,sha256=ARq6pJhvCGd79--IWUk0hq4b4eC-zDU8n4UWVRWLEAA,1202
-ragbits/evaluate/stores/base.py,sha256=PCtUFZWbPshcjxf3uhCylzTTK98qqnjp2Rw2LuipDro,2987
-ragbits/evaluate/stores/file.py,sha256=tz3N0oilQLLGylu6ADHT358kvHf1zGqvNVMWWdbejuw,18581
-ragbits/evaluate/stores/kv.py,sha256=WsmowNtTtvXf2Jjt5pSqAGbFt64KdFq4UXL9v5JnPpQ,21624
-ragbits_evaluate-0.0.30rc1.dist-info/METADATA,sha256=__OeAgqfwoHZuN3LqDXP4I7DEC-_12enk3UY_Y0lfOM,2553
-ragbits_evaluate-0.0.30rc1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-ragbits_evaluate-0.0.30rc1.dist-info/RECORD,,
+ragbits_evaluate-1.4.0.dev202602030301.dist-info/METADATA,sha256=XQ4uwmwNQTu4DUrfFrAdZLMiyRn_a1YlSkkiEuIcIZw,2368
+ragbits_evaluate-1.4.0.dev202602030301.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+ragbits_evaluate-1.4.0.dev202602030301.dist-info/RECORD,,