ragaai-catalyst 2.1.5b23__tar.gz → 2.1.5b25__tar.gz

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b23
+Version: 2.1.5b25
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: <3.13,>=3.9

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/examples/llamaindex_examples/joke_gen_critique_anthropic.py

@@ -1,4 +1,6 @@
-# !pip install llama-index-llms-anthropic
+import sys
+sys.path.append(".")
+
 from llama_index.core.workflow import (
     Event,
     StartEvent,
@@ -10,13 +12,11 @@ from llama_index.core.workflow import (
 from llama_index.llms.anthropic import Anthropic
 from dotenv import load_dotenv
 import os
-import sys
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
+
 
 load_dotenv()
-from ragaai_catalyst.tracers import Tracer
-from ragaai_catalyst import RagaAICatalyst, init_tracing
-from ragaai_catalyst import trace_llm
+
+from ragaai_catalyst import RagaAICatalyst, Tracer, init_tracing, trace_llm, current_span
 
 catalyst = RagaAICatalyst(
     access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
@@ -41,15 +41,46 @@ class JokeFlow(Workflow):
     llm = Anthropic()
 
     @step
-    #@trace_llm("generate joke")
+    @trace_llm("generate joke")
     async def generate_joke(self, ev: StartEvent) -> JokeEvent:
         topic = ev.topic
         prompt = f"Write your best joke about {topic}."
         response = await self.llm.acomplete(prompt)
+
+        current_span().add_metrics(
+            name="toxicity",
+            score=0.5,
+            reason="some reason"
+        )
+
+        # current_span().execute_metrics(
+        #     name="Hallucination",
+        #     model="gpt-4o-mini",
+        #     provider="openai",
+        #     display_name="Hallucination_display",
+        #     mapping={
+        #         'prompt': prompt,
+        #         'context': "travel agent",
+        #         'response': response
+        #     }
+        # )
+
+        # current_span().execute_metrics(
+        #     name="Hallucination",
+        #     model="gpt-4o-mini",
+        #     provider="openai",
+        #     display_name="Hallucination_display1",
+        #     mapping={
+        #         'prompt': prompt,
+        #         'context': "travel agent",
+        #         'response': response
+        #     }
+        # )
+
         return JokeEvent(joke=str(response))
 
     @step
-    #@trace_llm("criticise joke")
+    @trace_llm("criticise joke")
     async def critique_joke(self, ev: JokeEvent) -> StopEvent:
         joke = ev.joke
         prompt = f"Give a thorough analysis and critique of the following joke: {joke}"

ragaai_catalyst-2.1.5b25/examples/llamaindex_examples/joke_gen_critique_async.py

@@ -0,0 +1,106 @@
+import sys
+
+from ragaai_catalyst.tracers.distributed import trace_agent 
+sys.path.append('.')
+
+import os
+from llama_index.llms.openai import OpenAI
+from llama_index.core.workflow import (
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+from ragaai_catalyst import RagaAICatalyst, Tracer, init_tracing, trace_llm, current_span, trace_agent
+
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+catalyst = RagaAICatalyst(
+    access_key=os.getenv("RAGAAI_CATALYST_ACCESS_KEY"),
+    secret_key=os.getenv("RAGAAI_CATALYST_SECRET_KEY"),
+    base_url=os.getenv("RAGAAI_CATALYST_BASE_URL"),
+)
+
+# Initialize tracer
+tracer = Tracer(
+    project_name="Execute_Metric_Test1",
+    dataset_name="joke_generation_workflow_async1",
+    tracer_type="Agentic",
+)
+
+init_tracing(catalyst=catalyst, tracer=tracer)
+
+class JokeEvent(Event):
+    joke: str
+
+
+class JokeFlow(Workflow):
+    llm = OpenAI()
+
+    @step
+    @trace_agent("generate joke")
+    async def generate_joke(self, ev: StartEvent) -> JokeEvent:
+        topic = ev.topic
+        prompt = f"Write your best joke about {topic}."
+        
+        # Get the current span and store its attributes
+        span = current_span()
+        span.add_context(context="joke generation")
+        span.add_metrics(
+            name="toxicity",
+            score=0.4,
+            reasoning="Reason for toxicity",
+        )
+
+        # First execute_metrics call
+        span.execute_metrics(
+            name="Hallucination",
+            display_name="hallucination_generate_joke",
+            provider="openai",
+            model="gpt-4o-mini",
+            mapping={
+                "prompt": prompt,
+                "response": "vnvnvs",
+                "context": "Some Context"
+            }
+        )
+        
+        # Perform async operation
+        response = await self.llm.acomplete(prompt)
+        
+        # Get fresh span after async operation
+        span.execute_metrics(
+            name="Hallucination",
+            display_name="hallucination_generate_joke_2",
+            provider="openai",
+            model="gpt-4o-mini",
+            mapping={
+                "prompt": prompt,
+                "response": response.text,
+                "context": "Some Context"
+            }
+        )
+
+        return JokeEvent(joke=str(response))
+
+    @step
+    @trace_agent("criticise joke")
+    async def critique_joke(self, ev: JokeEvent) -> StopEvent:
+        joke = ev.joke
+        prompt = f"Give a thorough analysis and critique of the following joke: {joke}"
+        response = await self.llm.acomplete(prompt)
+        return StopEvent(result=str(response))
+    
+
+async def main():
+    w = JokeFlow(timeout=60, verbose=False)
+    result = await w.run(topic="climate change")
+    print(str(result))
+
+if __name__ == "__main__":
+    import asyncio
+    with tracer:
+        asyncio.run(main())

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/examples/travel_agent/config.py

@@ -15,7 +15,7 @@ def initialize_tracing():
     )
 
     tracer = Tracer(
-        project_name="swarnendu-4",
+        project_name="Execute_Metric_Test1",
         dataset_name="travel_agent_dataset",
         tracer_type="Agentic",
     )

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/examples/travel_agent/tools.py

@@ -34,6 +34,19 @@ def llm_call(prompt, max_tokens=512, model="gpt-4o-mini", name="default"):
         }
     )
 
+    current_span().execute_metrics(
+        name="Hallucination",
+        model="gpt-4o-mini",
+        provider="openai",
+        display_name="Hallucination_display1",
+        mapping={
+            'prompt': "goa to mumbai price",
+            'context': "travel agent",
+            'response': "approximately 10",
+            "gt": "10"
+        }
+    )
+
     response = client.chat.completions.create(
         model=model,
         messages=[{"role": "user", "content": prompt}],

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/pyproject.toml

@@ -8,8 +8,7 @@ description = "RAGA AI CATALYST"
 readme = "README.md"
 requires-python = ">=3.9,<3.13"
 # license = {file = "LICENSE"}
-
-version = "2.1.5.b23"
+version = "2.1.5.b25"
 authors = [
     {name = "Kiran Scaria", email = "kiran.scaria@raga.ai"},
     {name = "Kedar Gaikwad", email = "kedar.gaikwad@raga.ai"},

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/evaluation.py

@@ -4,6 +4,7 @@ import pandas as pd
 import io
 from .ragaai_catalyst import RagaAICatalyst
 import logging
+import json
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +19,7 @@ class Evaluation:
         self.project_name = project_name
         self.dataset_name = dataset_name
         self.base_url = f"{RagaAICatalyst.BASE_URL}"
-        self.timeout = 10
+        self.timeout = 20
         self.jobId = None
         self.num_projects=99999
 
@@ -357,6 +358,52 @@ class Evaluation:
         except Exception as e:
             logger.error(f"An unexpected error occurred: {e}")
 
+    def append_metrics(self, display_name):
+        if not isinstance(display_name, str):
+            raise ValueError("display_name should be a string")
+        
+        headers = {
+            "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
+            'X-Project-Id': str(self.project_id),
+            'Content-Type': 'application/json',
+        }
+
+        payload = json.dumps({
+        "datasetId": self.dataset_id,
+        "metricParams": [
+            {
+            "metricSpec": {
+                "displayName": display_name
+            }
+            }
+        ]
+        })
+        
+        try:
+            response = requests.request(
+                "POST", 
+                f'{self.base_url}/v2/llm/metric-evaluation-rerun', 
+                headers=headers, 
+                data=payload, 
+                timeout=self.timeout)
+            if response.status_code == 400:
+                raise ValueError(response.json()["message"])
+            response.raise_for_status()
+            if response.json()["success"]:
+                print(response.json()["message"])
+                self.jobId = response.json()["data"]["jobId"]
+
+        except requests.exceptions.HTTPError as http_err:
+            logger.error(f"HTTP error occurred: {http_err}")
+        except requests.exceptions.ConnectionError as conn_err:
+            logger.error(f"Connection error occurred: {conn_err}")
+        except requests.exceptions.Timeout as timeout_err:
+            logger.error(f"Timeout error occurred: {timeout_err}")
+        except requests.exceptions.RequestException as req_err:
+            logger.error(f"An error occurred: {req_err}")
+        except Exception as e:
+            logger.error(f"An unexpected error occurred: {e}")
+
     def get_status(self):
         headers = {
             'Content-Type': 'application/json',

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py

@@ -9,6 +9,7 @@ import contextvars
 import asyncio
 from ..utils.file_name_tracker import TrackName
 from ..utils.span_attributes import SpanAttributes
+from .base import BaseTracer
 import logging
 
 logger = logging.getLogger(__name__)
@@ -555,6 +556,10 @@ class AgentTracerMixin:
                 metrics.append(metric)
 
         # TODO agent_trace execute metric
+        formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
+        if formatted_metrics:
+            metrics.extend(formatted_metrics)
+
         component = {
             "id": kwargs["component_id"],
             "hash_id": kwargs["hash_id"],

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/tracers/base.py

@@ -1055,29 +1055,20 @@ class BaseTracer:
         return self.span_attributes_dict[span_name]
 
     @staticmethod
-    def get_formatted_metric(span_attributes_dict, project_id, name, prompt, span_context, response, span_gt):
+    def get_formatted_metric(span_attributes_dict, project_id, name):
         if name in span_attributes_dict:
             local_metrics = span_attributes_dict[name].local_metrics or []
+            local_metrics_results = []
             for metric in local_metrics:
                 try:
-                    if metric.get("prompt") is not None:
-                        prompt = metric['prompt']
-                    if metric.get("response") is not None:
-                        response = metric['response']
-                    if metric.get('context') is not None:
-                        span_context = metric['context']
-                    if metric.get('gt') is not None:
-                        span_gt = metric['gt']
-
                     logger.info("calculating the metric, please wait....")
+
+                    mapping = metric.get("mapping", {})
                     result = calculate_metric(project_id=project_id,
                                               metric_name=metric.get("name"),
                                               model=metric.get("model"),
                                               provider=metric.get("provider"),
-                                              prompt=prompt,
-                                              context=span_context,
-                                              response=response,
-                                              expected_response=span_gt
+                                              **mapping
                                               )
 
                     result = result['data']['data'][0]
@@ -1107,9 +1098,11 @@ class BaseTracer:
                         "mappings": [],
                         "config": metric_config
                     }
-                    return formatted_metric
+                    local_metrics_results.append(formatted_metric)
                 except ValueError as e:
                     logger.error(f"Validation Error: {e}")
                 except Exception as e:
                     logger.error(f"Error executing metric: {e}")
 
+            return local_metrics_results
+

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py

@@ -630,9 +630,9 @@ class LLMTracerMixin:
         #print("Response output: ",response)
 
         # TODO: Execute & Add the User requested metrics here
-        formatted_metric = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name, prompt, span_context, response, span_gt)
-        if formatted_metric is not None:
-            metrics.append(formatted_metric)
+        formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
+        if formatted_metrics:
+            metrics.extend(formatted_metrics)
 
         component = {
             "id": component_id,
@@ -683,38 +683,36 @@ class LLMTracerMixin:
         # return "\n".join(process_content(msg.get("content", "")) for msg in messages if msg.get("content"))
   
     def convert_to_content(self, input_data):
-        if isinstance(input_data, dict):
-            messages = input_data.get("kwargs", {}).get("messages", [])
-        elif isinstance(input_data, list):
-            if len(input_data)>0 and isinstance(input_data[0]['content'],ChatResponse):
-                extracted_messages = []
-
-                for item in input_data:
-                    chat_response = item.get('content')
-                    if hasattr(chat_response, 'message') and hasattr(chat_response.message, 'blocks'):
-                        for block in chat_response.message.blocks:
-                            if hasattr(block, 'text'):
-                                extracted_messages.append(block.text)
-                messages=extracted_messages
-                if isinstance(messages,list):
-                    return "\n".join(messages)
-                
-                #messages=[msg["content"] for msg in input_data if isinstance(msg, dict) and "content" in msg]
-                #messages = [msg["content"].message for msg in input_data if isinstance(msg, dict) and "content" in msg and isinstance(msg["content"], ChatResponse)]
+        try:
+            if isinstance(input_data, dict):
+                messages = input_data.get("kwargs", {}).get("messages", [])
+            elif isinstance(input_data, list):
+                if len(input_data)>0 and isinstance(input_data[0]['content'],ChatResponse):
+                    extracted_messages = []
+
+                    for item in input_data:
+                        chat_response = item.get('content')
+                        if hasattr(chat_response, 'message') and hasattr(chat_response.message, 'blocks'):
+                            for block in chat_response.message.blocks:
+                                if hasattr(block, 'text'):
+                                    extracted_messages.append(block.text)
+                    messages=extracted_messages
+                    if isinstance(messages,list):
+                        return "\n".join(messages)
+                    
+                    #messages=[msg["content"] for msg in input_data if isinstance(msg, dict) and "content" in msg]
+                    #messages = [msg["content"].message for msg in input_data if isinstance(msg, dict) and "content" in msg and isinstance(msg["content"], ChatResponse)]
+                else:
+                    messages = input_data
+            elif isinstance(input_data,ChatResponse):
+                messages=input_data['content']
             else:
-                messages = input_data
-        elif isinstance(input_data,ChatResponse):
-            messages=input_data['content']
-        else:
-            return ""
-        res=""
-        # try:
-        res="\n".join(msg.get("content", "").strip() for msg in messages if msg.get("content"))
-        # except Exception as e:
-        #     print("Exception occured for: ",e)
-        #     print("Input: ",input_data,"Meeage: ",messages)
-        #     # import sys
-        #     # sys.exit()
+                return ""
+            res=""
+            # try:
+            res="\n".join(msg.get("content", "").strip() for msg in messages if msg.get("content"))
+        except Exception as e:
+            res=str(messages)
         return res
 
     def process_content(content):

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py

@@ -361,7 +361,7 @@ class AgenticTracing(
 
         # Check if there's an active agent context
         current_agent_id = self.current_agent_id.get()
-        if current_agent_id and component_data["type"] in ["llm", "tool"]:
+        if current_agent_id and component_data["type"] in ["llm", "tool", "custom"]:
             # Add this component as a child of the current agent
             current_children = self.agent_children.get()
             current_children.append(component_data)

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py

@@ -7,6 +7,7 @@ import functools
 from typing import Optional, Any, Dict, List
 
 from pydantic import tools
+from .base import BaseTracer
 from ..utils.unique_decorator import generate_unique_hash_simple
 import contextvars
 import asyncio
@@ -483,6 +484,10 @@ class ToolTracerMixin:
                 metric["name"] = metric_name  
                 metrics.append(metric)
 
+        formatted_metrics = BaseTracer.get_formatted_metric(self.span_attributes_dict, self.project_id, name)
+        if formatted_metrics:
+            metrics.extend(formatted_metrics)
+
         start_time = kwargs["start_time"]
         component = {
             "id": kwargs["component_id"],

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py

@@ -11,13 +11,14 @@ logging_level = (
     else logger.setLevel(logging.INFO)
 )
 
-def calculate_metric(project_id, metric_name, model, provider, prompt, response, context, expected_response=None):
+
+def calculate_metric(project_id, metric_name, model, provider, **kwargs):
     user_id = "1"
     org_domain = "raga"
 
     headers = {
         "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-        "X-Project-Id": str(project_id),  # TODO to change it to project_id
+        "X-Project-Id": str(project_id),
         "Content-Type": "application/json"
     }
 
@@ -38,18 +39,19 @@ def calculate_metric(project_id, metric_name, model, provider, prompt, response,
                     "metric_name": metric_name,
                     "request_id": 1
                 },
+                "variable_mapping": kwargs,
                 "trace_object": {
                     "Data": {
                         "DocId": "doc-1",
-                        "Prompt": prompt,
-                        "Response": response,
-                        "Context": context,
-                        "ExpectedResponse": "",
-                        "ExpectedContext": expected_response,
-                        "Chat": "",
-                        "Instructions": "",
-                        "SystemPrompt": "",
-                        "Text": ""
+                        "Prompt": kwargs.get("prompt"),
+                        "Response": kwargs.get("response"),
+                        "Context": kwargs.get("context"),
+                        "ExpectedResponse": kwargs.get("expected_response"),
+                        "ExpectedContext": kwargs.get("expected_context"),
+                        "Chat": kwargs.get("chat"),
+                        "Instructions": kwargs.get("instructions"),
+                        "SystemPrompt": kwargs.get("system_prompt"),
+                        "Text": kwargs.get("text")
                     },
                     "claims": {},
                     "last_computed_metrics": {

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py

@@ -62,6 +62,7 @@ class SpanAttributes:
         self.feedback = feedback
         logger.debug(f"Added feedback: {self.feedback}")
 
+    # TODO: Add validation to check if all the required parameters are present
     def execute_metrics(self, **kwargs: Any):
         name = kwargs.get("name")
         model = kwargs.get("model")
@@ -91,19 +92,20 @@ class SpanAttributes:
             prompt =None
             context = None
             response = None
-            if mapping is not None:
-                prompt = mapping['prompt']
-                context = mapping['context']
-                response = mapping['response']
+            # if mapping is not None:
+            #     prompt = mapping['prompt']
+            #     context = mapping['context']
+            #     response = mapping['response']
             new_metric = {
                 "name": metric_name,
                 "model": model,
                 "provider": provider,
                 "project_id": self.project_id,
-                "prompt": prompt,
-                "context": context,
-                "response": response,
-                "displayName": display_name
+                # "prompt": prompt,
+                # "context": context,
+                # "response": response,
+                "displayName": display_name,
+                "mapping": mapping
             }
             self.local_metrics.append(new_metric)
 

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b23
+Version: 2.1.5b25
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
 Requires-Python: <3.13,>=3.9

{ragaai_catalyst-2.1.5b23 → ragaai_catalyst-2.1.5b25}/ragaai_catalyst.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ examples/langgraph_examples/planning_agent.py
 examples/llamaindex_examples/function_calling_agent.ipynb
 examples/llamaindex_examples/joke_gen_critique.py
 examples/llamaindex_examples/joke_gen_critique_anthropic.py
+examples/llamaindex_examples/joke_gen_critique_async.py
 examples/llamaindex_examples/joke_gen_critique_azureopenai.py
 examples/llamaindex_examples/joke_gen_critique_gemini.py
 examples/llamaindex_examples/joke_gen_critique_litellm.py