aiqtoolkit 1.2.0a20250525__py3-none-any.whl → 1.2.0a20250526__py3-none-any.whl

aiq/eval/intermediate_step_adapter.py

@@ -79,15 +79,21 @@ class IntermediateStepAdapter:
         for step in steps:
             if step.event_type == IntermediateStepType.LLM_END:
                 last_llm_end_step = step
+                action = self.get_agent_action_single(step, "")
+                agent_actions.append(action)
             else:
                 action = self.get_agent_action_single(step, last_llm_end_step)
                 agent_actions.append(action)
 
         return agent_actions
 
-    def get_context(self, intermediate_steps: list[IntermediateStep]) -> list[str]:
+    def get_context(self, intermediate_steps: list[IntermediateStep],
+                    event_filter: list[IntermediateStepType]) -> list[str]:
         """Grab the output of all the tools and return them as retrieved context."""
-        return [
-            str(step.data.output) for step in intermediate_steps
-            if step.event_type == IntermediateStepType.TOOL_END and step.data and step.data.output
-        ]
+        count = 0
+        agent_actions = []
+        for step in intermediate_steps:
+            if step.event_type in event_filter and step.data and step.data.output:
+                agent_actions.append(f"**Step {count}**\n{str(step.data.output)}")
+                count += 1
+        return agent_actions

aiq/eval/rag_evaluator/evaluate.py

@@ -23,6 +23,7 @@ from ragas.llms import LangchainLLMWrapper
 from ragas.metrics import Metric
 from tqdm import tqdm
 
+from aiq.data_models.intermediate_step import IntermediateStepType
 from aiq.eval.evaluator.evaluator_model import EvalInput
 from aiq.eval.evaluator.evaluator_model import EvalOutput
 from aiq.eval.evaluator.evaluator_model import EvalOutputItem
@@ -33,15 +34,16 @@ logger = logging.getLogger(__name__)
 
 class RAGEvaluator:
 
-    def __init__(self, evaluator_llm: LangchainLLMWrapper, metrics: Sequence[Metric]):
+    def __init__(self, evaluator_llm: LangchainLLMWrapper, metrics: Sequence[Metric], max_concurrency=8):
         self.evaluator_llm = evaluator_llm
         self.metrics = metrics
+        self.max_concurrency = max_concurrency
 
     @staticmethod
     def eval_input_to_ragas(eval_input: EvalInput) -> EvaluationDataset:
         """Converts EvalInput into a Ragas-compatible EvaluationDataset."""
         from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
-
+        event_filter = [IntermediateStepType.TOOL_END, IntermediateStepType.LLM_END, IntermediateStepType.CUSTOM_END]
         samples = []
 
         intermediate_step_adapter = IntermediateStepAdapter()
@@ -55,7 +57,7 @@ class RAGEvaluator:
             reference_contexts = [""]  # Default to empty context
             # implement context extraction from expected_trajectory
 
-            retrieved_contexts = intermediate_step_adapter.get_context(item.trajectory)
+            retrieved_contexts = intermediate_step_adapter.get_context(item.trajectory, event_filter)
             # implement context extraction from expected_trajectory
 
             # Create a SingleTurnSample
@@ -116,6 +118,7 @@ class RAGEvaluator:
     async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
         """Run Ragas metrics evaluation on the provided EvalInput"""
         from ragas import evaluate as ragas_evaluate
+        from ragas.run_config import RunConfig
 
         ragas_dataset = self.eval_input_to_ragas(eval_input)
         tqdm_position = TqdmPositionRegistry.claim()
@@ -126,6 +129,7 @@ class RAGEvaluator:
                                              metrics=self.metrics,
                                              show_progress=True,
                                              llm=self.evaluator_llm,
+                                             run_config=RunConfig(max_workers=self.max_concurrency),
                                              _pbar=pbar)
         except Exception as e:
             # On exception we still continue with other evaluators. Log and return an avg_score of 0.0

aiq/eval/rag_evaluator/register.py

@@ -133,6 +133,7 @@ async def register_ragas_evaluator(config: RagasEvaluatorConfig, builder: EvalBu
             metrics.append(metric_callable(**kwargs))
 
     # Create the RAG evaluator
-    _evaluator = RAGEvaluator(evaluator_llm=llm, metrics=metrics) if metrics else None
+    _evaluator = RAGEvaluator(evaluator_llm=llm, metrics=metrics,
+                              max_concurrency=builder.get_max_concurrency()) if metrics else None
 
     yield EvaluatorInfo(config=config, evaluate_fn=evaluate_fn, description="Evaluator for RAGAS metrics")

{aiqtoolkit-1.2.0a20250525.dist-info → aiqtoolkit-1.2.0a20250526.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiqtoolkit
-Version: 1.2.0a20250525
+Version: 1.2.0a20250526
 Summary: NVIDIA Agent Intelligence toolkit
 Author: NVIDIA Corporation
 Maintainer: NVIDIA Corporation

{aiqtoolkit-1.2.0a20250525.dist-info → aiqtoolkit-1.2.0a20250526.dist-info}/RECORD

@@ -108,7 +108,7 @@ aiq/embedder/register.py,sha256=3MTZrfNQKp6AZTbfaA-PpTnyXiMyu-8HH9JnDCC0v9o,978
 aiq/eval/__init__.py,sha256=Xs1JQ16L9btwreh4pdGKwskffAw1YFO48jKrU4ib_7c,685
 aiq/eval/config.py,sha256=IlOr2o618kbkXP0G1F-AklZfsKYVos9UB4Dvlxf66xk,1431
 aiq/eval/evaluate.py,sha256=WPGLBeJ46mwIlnprbtia1cm2MwMqZ-GskXoTn6R4oV0,14624
-aiq/eval/intermediate_step_adapter.py,sha256=D645cfnncBEYc-LhYNwr0NwweCr1kTLENt1pLNvmVvU,4211
+aiq/eval/intermediate_step_adapter.py,sha256=4cSsGgFBvNjXnclk5FvZnQaFEdeulp7VEdRWKLcREAQ,4498
 aiq/eval/register.py,sha256=QOHJqA2CQixeWMC9InyKbzXo1jByvrntD_m9-2Mvg9k,1076
 aiq/eval/remote_workflow.py,sha256=Fb7Z6gdP2L_gqyWB--AEWfcXe9xPpQ_hPsf9lmqGXjI,5524
 aiq/eval/runtime_event_subscriber.py,sha256=2VM8MqmPc_EWPxxrDDR9naiioZirkJUfGwzbXQqbdZA,1906
@@ -119,8 +119,8 @@ aiq/eval/dataset_handler/dataset_handler.py,sha256=cqdGVgHm6tsKk3TwFcFhptxAvcHVl
 aiq/eval/evaluator/__init__.py,sha256=GUJrgGtpvyMUCjUBvR3faAdv-tZzbU9W-izgx9aMEQg,680
 aiq/eval/evaluator/evaluator_model.py,sha256=alO8bVoGmvej1LpN5wZ5HG29TSrL4IMWdVcMew8IOzM,1405
 aiq/eval/rag_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aiq/eval/rag_evaluator/evaluate.py,sha256=sxOVHxSOSYvTWFV9jOTpGDrwf2PwkuI3iu_cFj5XbgU,6136
-aiq/eval/rag_evaluator/register.py,sha256=1SgbdMmMvk77Kh6BhP-XvWO2_8WEHEDtcHyoL8qwY_E,5565
+aiq/eval/rag_evaluator/evaluate.py,sha256=lEjXKiuELAHyWckz-bM91dZ6AT2J6NC7SfvtedR-Qdk,6548
+aiq/eval/rag_evaluator/register.py,sha256=2NzxkgqyoZ4wC8ARj3tiVoE8ENCmplBCIKrNOFh6_VI,5642
 aiq/eval/swe_bench_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/eval/swe_bench_evaluator/evaluate.py,sha256=kNukRruq1EM1RsGLvpVuC22xcP0gpn9acF3edGak9vY,9858
 aiq/eval/swe_bench_evaluator/register.py,sha256=sTb74F7w4iuI0ROsEJ4bV13Nt1GEWQn7UvO2O0HXwXk,1537
@@ -307,10 +307,10 @@ aiq/utils/reactive/base/observer_base.py,sha256=UAlyAY_ky4q2t0P81RVFo2Bs_R7z5Nde
 aiq/utils/reactive/base/subject_base.py,sha256=Ed-AC6P7cT3qkW1EXjzbd5M9WpVoeN_9KCe3OM3FLU4,2521
 aiq/utils/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiq/utils/settings/global_settings.py,sha256=U9TCLdoZsKq5qOVGjREipGVv9e-FlStzqy5zv82_VYk,7454
-aiqtoolkit-1.2.0a20250525.dist-info/licenses/LICENSE-3rd-party.txt,sha256=8o7aySJa9CBvFshPcsRdJbczzdNyDGJ8b0J67WRUQ2k,183936
-aiqtoolkit-1.2.0a20250525.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-aiqtoolkit-1.2.0a20250525.dist-info/METADATA,sha256=VtrrrJKL0ZyCGxF6LN-DCjzOMOUl1YcmDH0WtWGgzMc,20174
-aiqtoolkit-1.2.0a20250525.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-aiqtoolkit-1.2.0a20250525.dist-info/entry_points.txt,sha256=gRlPfR5g21t328WNEQ4CcEz80S1sJNS8A7rMDYnzl4A,452
-aiqtoolkit-1.2.0a20250525.dist-info/top_level.txt,sha256=fo7AzYcNhZ_tRWrhGumtxwnxMew4xrT1iwouDy_f0Kc,4
-aiqtoolkit-1.2.0a20250525.dist-info/RECORD,,
+aiqtoolkit-1.2.0a20250526.dist-info/licenses/LICENSE-3rd-party.txt,sha256=8o7aySJa9CBvFshPcsRdJbczzdNyDGJ8b0J67WRUQ2k,183936
+aiqtoolkit-1.2.0a20250526.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+aiqtoolkit-1.2.0a20250526.dist-info/METADATA,sha256=OERp4WxL1pg6wyXVwNFtsG7a6rbpdcpcuFgK8Bx56sg,20174
+aiqtoolkit-1.2.0a20250526.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+aiqtoolkit-1.2.0a20250526.dist-info/entry_points.txt,sha256=gRlPfR5g21t328WNEQ4CcEz80S1sJNS8A7rMDYnzl4A,452
+aiqtoolkit-1.2.0a20250526.dist-info/top_level.txt,sha256=fo7AzYcNhZ_tRWrhGumtxwnxMew4xrT1iwouDy_f0Kc,4
+aiqtoolkit-1.2.0a20250526.dist-info/RECORD,,