ragaai-catalyst 2.1.6.4b0__py3-none-any.whl → 2.1.7__py3-none-any.whl

ragaai_catalyst/tracers/utils/rag_trace_json_converter.py

@@ -0,0 +1,243 @@
+import json
+from litellm import model_cost
+import logging
+import os
+import re
+from datetime import datetime
+import tiktoken
+
+logger = logging.getLogger("RagaAICatalyst")
+logging_level = (
+    logger.setLevel(logging.DEBUG) if os.getenv("DEBUG") == "1" else logging.INFO
+)
+
+def rag_trace_json_converter(input_trace, custom_model_cost, trace_id, user_details, tracer_type):
+    trace_aggregate = {}
+    
+    def get_prompt(input_trace):
+        if tracer_type == "langchain":
+            for span in input_trace:
+                if span["name"] in ["ChatOpenAI", "ChatAnthropic", "ChatGoogleGenerativeAI"]:
+                    return span["attributes"].get("llm.input_messages.1.message.content")
+
+                elif span["name"] == "LLMChain":
+                    return json.loads(span["attributes"].get("input.value", "{}")).get("question")
+
+                elif span["name"] == "RetrievalQA":
+                    return span["attributes"].get("input.value")
+                
+                elif span["name"] == "VectorStoreRetriever":
+                    return span["attributes"].get("input.value")
+                
+        return None
+    
+    def get_response(input_trace):
+        if tracer_type == "langchain":
+            for span in input_trace:
+                if span["name"] in ["ChatOpenAI", "ChatAnthropic", "ChatGoogleGenerativeAI"]:
+                    return span["attributes"].get("llm.output_messages.0.message.content")
+
+                elif span["name"] == "LLMChain":
+                    return json.loads(span["attributes"].get("output.value", ""))
+
+                elif span["name"] == "RetrievalQA":
+                    return span["attributes"].get("output.value")
+
+        return None
+    
+    def get_context(input_trace):
+        if tracer_type == "langchain":
+            for span in input_trace:
+                if span["name"] == "VectorStoreRetriever":
+                    return span["attributes"].get("retrieval.documents.1.document.content")
+        return None
+        
+    prompt = get_prompt(input_trace)
+    response = get_response(input_trace)
+    context = get_context(input_trace)
+    
+    if tracer_type == "langchain":
+        trace_aggregate["tracer_type"] = "langchain"
+    else:
+        trace_aggregate["tracer_type"] = "llamaindex"
+
+    trace_aggregate['trace_id'] = trace_id
+    trace_aggregate['session_id'] = None
+    trace_aggregate["metadata"] = user_details.get("trace_user_detail", {}).get("metadata")
+
+    #dummy data need to be fetched
+    trace_aggregate["pipeline"] = {
+        'llm_model': 'gpt-4o-mini', 
+        'vector_store': 'faiss',
+        'embed_model': 'text-embedding-ada-002'
+        }
+    
+    trace_aggregate["data"] = {}
+    trace_aggregate["data"]["prompt"] = prompt
+    trace_aggregate["data"]["response"] = response
+    trace_aggregate["data"]["context"] = context
+    
+    if tracer_type == "langchain":
+        additional_metadata = get_additional_metadata(input_trace, custom_model_cost, model_cost, prompt, response)
+    else:
+        additional_metadata = get_additional_metadata(input_trace, custom_model_cost, model_cost)
+    
+    trace_aggregate["metadata"].update(additional_metadata)
+    additional_metadata.pop("total_cost")
+    additional_metadata.pop("total_latency")
+    return trace_aggregate, additional_metadata
+
+def get_additional_metadata(spans, custom_model_cost, model_cost_dict, prompt="", response=""):
+    additional_metadata = {}
+    additional_metadata["cost"] = 0.0
+    additional_metadata["tokens"] = {}
+    try:
+        for span in spans:
+            if span["name"] in ["ChatOpenAI", "ChatAnthropic", "ChatGoogleGenerativeAI"]:
+                start_time = datetime.fromisoformat(span.get("start_time", "")[:-1])  # Remove 'Z' and parse
+                end_time = datetime.fromisoformat(span.get("end_time", "")[:-1])    # Remove 'Z' and parse
+                additional_metadata["latency"] = (end_time - start_time).total_seconds()
+                additional_metadata["model_name"] = span["attributes"].get("llm.model_name", "").replace("models/", "")
+                additional_metadata["model"] = additional_metadata["model_name"]
+                try:
+                    additional_metadata["tokens"]["prompt"] = span["attributes"]["llm.token_count.prompt"]
+
+                except:
+                    logger.warning("Warning: prompt token not found. using fallback strategies to get tokens.")
+                    try:
+                        additional_metadata["tokens"]["prompt"] = num_tokens_from_messages(
+                            model=additional_metadata["model_name"],
+                            message=prompt
+                        )
+                    except Exception as e:
+                        logger.warning(f"Failed to count prompt tokens: {str(e)}. Using 'gpt-4o-mini' model count as fallback.")
+                        additional_metadata["tokens"]["prompt"] = num_tokens_from_messages(
+                            model="gpt-4o-mini",
+                            message=prompt
+                        )
+                
+                try:
+                    additional_metadata["tokens"]["completion"] = span["attributes"]["llm.token_count.completion"]
+                except:
+                    logger.warning("Warning: completion token not found. using fallback strategies to get tokens.")
+                    try:
+                        additional_metadata["tokens"]["completion"] = num_tokens_from_messages(
+                            model=additional_metadata["model_name"],
+                            message=response
+                        )
+                    except Exception as e:
+                        logger.warning(f"Failed to count completion tokens: {str(e)}. Using 'gpt-4o-mini' model count as fallback.")
+                        additional_metadata["tokens"]["completion"] = num_tokens_from_messages(
+                            model="gpt-4o-mini",
+                            message=response
+                        )
+                
+                # Ensure both values are not None before adding
+                prompt_tokens = additional_metadata["tokens"].get("prompt", 0) or 0
+                completion_tokens = additional_metadata["tokens"].get("completion", 0) or 0
+                additional_metadata["tokens"]["total"] = prompt_tokens + completion_tokens
+
+    except Exception as e:
+        logger.error(f"Error getting additional metadata: {str(e)}")
+    
+    try:
+        if custom_model_cost.get(additional_metadata.get('model_name')):
+            model_cost_data = custom_model_cost[additional_metadata.get('model_name')]
+        else:
+            model_cost_data = model_cost_dict.get(additional_metadata.get('model_name'))
+        
+        # Check if model_cost_data is None
+        if model_cost_data is None:
+            logger.warning(f"No cost data found for model: {additional_metadata.get('model_name')}")
+            # Set default values
+            additional_metadata["cost"] = 0.0
+            additional_metadata["total_cost"] = 0.0
+            additional_metadata["total_latency"] = additional_metadata.get("latency", 0)
+            additional_metadata["prompt_tokens"] = additional_metadata["tokens"].get("prompt", 0) or 0
+            additional_metadata["completion_tokens"] = additional_metadata["tokens"].get("completion", 0) or 0
+        elif 'tokens' in additional_metadata and all(k in additional_metadata['tokens'] for k in ['prompt', 'completion']):
+            # Get input and output costs, defaulting to 0 if not found
+            input_cost_per_token = model_cost_data.get("input_cost_per_token", 0) or 0
+            output_cost_per_token = model_cost_data.get("output_cost_per_token", 0) or 0
+            
+            # Get token counts, defaulting to 0 if not found
+            prompt_tokens = additional_metadata["tokens"].get("prompt", 0) or 0
+            completion_tokens = additional_metadata["tokens"].get("completion", 0) or 0
+            
+            # Calculate costs
+            prompt_cost = prompt_tokens * input_cost_per_token
+            completion_cost = completion_tokens * output_cost_per_token
+            
+            additional_metadata["cost"] = prompt_cost + completion_cost 
+            additional_metadata["total_cost"] = additional_metadata["cost"]
+            additional_metadata["total_latency"] = additional_metadata.get("latency", 0)
+            additional_metadata["prompt_tokens"] = prompt_tokens
+            additional_metadata["completion_tokens"] = completion_tokens
+    except Exception as e:
+        logger.warning(f"Error getting model cost data: {str(e)}")
+        # Set default values in case of error
+        additional_metadata["cost"] = 0.0
+        additional_metadata["total_cost"] = 0.0
+        additional_metadata["total_latency"] = additional_metadata.get("latency", 0)
+        additional_metadata["prompt_tokens"] = additional_metadata["tokens"].get("prompt", 0) or 0
+        additional_metadata["completion_tokens"] = additional_metadata["tokens"].get("completion", 0) or 0
+    try:
+        additional_metadata.pop("tokens", None)
+    except Exception as e:
+        logger.error(f"Error removing tokens from additional metadata: {str(e)}")
+
+    return additional_metadata
+
+def num_tokens_from_messages(model, message):
+    # GPT models
+    if re.match(r'^gpt-', model):
+        """Check if the model is any GPT model (pattern: ^gpt-)
+        This matches any model name that starts with 'gpt-'
+        """
+        def num_tokens_from_string(string: str, encoding_name: str) -> int:
+            """Returns the number of tokens in a text string."""
+            encoding = tiktoken.get_encoding(encoding_name)
+            num_tokens = len(encoding.encode(string))
+            return num_tokens
+        
+        if re.match(r'^gpt-4o.*', model):
+            """Check for GPT-4 Optimized models (pattern: ^gpt-4o.*)
+            Examples that match:
+            - gpt-4o
+            - gpt-4o-mini
+            - gpt-4o-2024-08-06
+            The .* allows for any characters after 'gpt-4o'
+            """
+            encoding_name = "o200k_base"
+            return num_tokens_from_string(message, encoding_name)
+        
+        elif re.match(r'^gpt-(4|3\.5).*', model):
+            """Check for GPT-4 and GPT-3.5 models (pattern: ^gpt-(4|3\.5).*)
+            Uses cl100k_base encoding for GPT-4 and GPT-3.5 models
+            Examples that match:
+            - gpt-4
+            - gpt-4-turbo
+            - gpt-4-2024-08-06
+            - gpt-3.5-turbo
+            - gpt-3.5-turbo-16k
+            """
+            encoding_name = "cl100k_base"
+            return num_tokens_from_string(message, encoding_name)
+        
+        else:
+            """Default case for any other GPT models
+            Uses o200k_base encoding as the default tokenizer
+            """
+            return num_tokens_from_string(message, encoding_name="o200k_base")
+        
+
+    # Gemini models 
+    elif re.match(r'^gemini-', model):
+        GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+        client = genai.Client(api_key=GOOGLE_API_KEY)
+
+        response = client.models.count_tokens(
+                model=model,
+                contents=message,
+            )
+        return response.total_tokens

ragaai_catalyst/tracers/utils/trace_json_converter.py

@@ -296,6 +296,7 @@ def convert_json_format(input_trace, custom_model_cost):
     final_trace["metadata"]["total_tokens"] = final_trace["metadata"]["tokens"]["total_tokens"]
 
     return final_trace
+
     
 if __name__ == "__main__":
     if len(sys.argv) != 3:

{ragaai_catalyst-2.1.6.4b0.dist-info → ragaai_catalyst-2.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragaai_catalyst
-Version: 2.1.6.4b0
+Version: 2.1.7
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
 Requires-Python: <3.13,>=3.10

{ragaai_catalyst-2.1.6.4b0.dist-info → ragaai_catalyst-2.1.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
 ragaai_catalyst/__init__.py,sha256=2wfkucAbb3Bt_p2KHelkg9zBQp4yC4iZanltyieG18w,895
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
-ragaai_catalyst/dataset.py,sha256=YCj8Ovu6y38KEw-1HCe4xQWkmYPgfNTtMa8Q0g6B62o,29401
+ragaai_catalyst/dataset.py,sha256=LefpZDCTkLoimZr0GEGK2awjOjhy8zSU-29EtqLrSG0,29404
 ragaai_catalyst/evaluation.py,sha256=O96CydYVPh3duUmXjY6REIXMOR-tOPixSG-Qhrf636A,22955
 ragaai_catalyst/experiment.py,sha256=8yQo1phCHlpnJ-4CqCaIbLXg_1ZlAuLGI9kqGBl-OTE,18859
 ragaai_catalyst/guard_executor.py,sha256=f2FXQSW17z4-eor61J_mtD0z-xBm9yordq8giB-GN_U,14006
@@ -31,8 +31,8 @@ ragaai_catalyst/tracers/distributed.py,sha256=MwlBwIxCAng-OI-7Ove_rkE1mTLeuW4Jw-
 ragaai_catalyst/tracers/langchain_callback.py,sha256=CB75zzG3-DkYTELj0vI1MOHQTY0MuQJfoHIXz9Cl8S8,34568
 ragaai_catalyst/tracers/llamaindex_callback.py,sha256=ZY0BJrrlz-P9Mg2dX-ZkVKG3gSvzwqBtk7JL_05MiYA,14028
 ragaai_catalyst/tracers/llamaindex_instrumentation.py,sha256=Ys_jLkvVqo12bKgXDmkp4TxJu9HkBATrFE8cIcTYxWw,14329
-ragaai_catalyst/tracers/tracer.py,sha256=JgiWCf43XwAl5sWyegqpFfS9c4EDbPWPZUy3ymOE0tw,37310
-ragaai_catalyst/tracers/upload_traces.py,sha256=alizdRRDobeeEPwgNIFwPfSFJvAo5I0W9CBTfUqzAzA,5887
+ragaai_catalyst/tracers/tracer.py,sha256=YZB3l55EXLog_rrAMb2jc5pvhQ63qtCc9UZcY7Yqmxo,37134
+ragaai_catalyst/tracers/upload_traces.py,sha256=PEE_JhAmOAMKyb-pl4ZoFWhIePxJm1zs93crrk94iEg,5887
 ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
 ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
 ragaai_catalyst/tracers/agentic_tracing/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -45,11 +45,11 @@ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU
 ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
 ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=LzbsHvELwBmH8ObFomJRhiQ98b6MEi18irm0DPiplt0,29743
-ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=1EmSvsSV_7Az4t6MBVYZcbllyYukYDZL6h40v2o31sU,54576
+ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=wYdGud8JKBnr78fizMGwTdA8qWRMpQyA2Il7tS3rSRU,55823
 ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=OBJJjFSvwRjCGNJyqX3yIfC1W05ZN2QUXasCJ4gmCjQ,13930
 ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=z-qzmCQCkhyW0aLDUR_rNq4pmxhAaVhNY-kZQsox-Ws,50221
-ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=9Kn6gppITCJ8obLyAGgfpWOlyqKfx2Zd4RkkMZMHYn8,16166
+ragaai_catalyst/tracers/agentic_tracing/tracers/main_tracer.py,sha256=Va66pvibVSgrr_Wfokog4toxLZw40SqxAmR_NZlWu70,16383
 ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl7iMiFya_lNwN1ykBc3Pmo-2pR_2HmpptwHWQ,10352
 ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=xxrliKPfdfbIZRZqMnUewsaTD8_Hv0dbuoBivNZGD4U,21674
 ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py,sha256=bhSUhNQCuJXKjgJAXhjKEYjnHMpYN90FSZdR84fNIKU,4614
@@ -70,25 +70,26 @@ ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=2tzGw_cKCT
 ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=qmODERcFZhc8MX24boFCXkkh6sJ-vZngRHPvxhyWFeE,4347
 ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml,sha256=LvFDivDIE96Zasp-fgDEqUJ5GEQZUawQucR3aOcSUTY,926
 ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py,sha256=H8WNsk4v_5T6OUw4TFOzlDLjQhJwjh1nAMyMAoqMEi4,6946
-ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=rssHolDvKxZ9V6-4VTFAqC65o6-CG924hA0CnG3smSc,15902
+ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=W7Nw-IpugejIoHbCtQiN4Sn4ughLocQ9AUCjuAtOhOo,17258
 ragaai_catalyst/tracers/agentic_tracing/utils/unique_decorator.py,sha256=G027toV-Km20JjKrc-Y_PilQ8ABEKrBvvzgLTnqVg7I,5819
 ragaai_catalyst/tracers/agentic_tracing/utils/zip_list_of_unique_files.py,sha256=4TeCGsFF26249fV6dJHLTZDrRa93SG9oer4rudoF8Y4,19443
 ragaai_catalyst/tracers/exporters/__init__.py,sha256=wQbaqyeIjVZxYprHCKZ9BeiqxeXYBKjzEgP79LWNxCU,293
-ragaai_catalyst/tracers/exporters/dynamic_trace_exporter.py,sha256=oFIubzCCmC3fpoH_ybohevjOE5WIEG9ahcPIEtwn99c,5093
+ragaai_catalyst/tracers/exporters/dynamic_trace_exporter.py,sha256=eR_bbWMg_q8g9SzutrOZA24Bptr0BWpauZKVWfiCM1c,5910
 ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=RgGteu-NVGprXKkynvyIO5yOjpbtA41R3W_NzCjnkwE,6445
 ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=6xvjWXyh8XPkHKSLLmAZUQSvwuyY17ov8pv2VdfI0qA,17875
-ragaai_catalyst/tracers/exporters/ragaai_trace_exporter.py,sha256=n78Yky-32yVI8fiL_3yD-iShgizb8IcN8OqQ5gOwZ9w,5387
+ragaai_catalyst/tracers/exporters/ragaai_trace_exporter.py,sha256=jfJr836ahLyo2qYrP5Hmdx0XkZt8lEhlKGHIP7PNmGs,11463
 ragaai_catalyst/tracers/instrumentators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
-ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py,sha256=ofrNrxf2b1hpjDh_zeaxiYq86azn1MF3kW8-ViYPEg0,1641
+ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py,sha256=e0URuRWCdzpxuBLfL82FOTMjbRuDAkW8aIRi7s7Nocc,1655
 ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py,sha256=8qLo7x4Zsn3dhJfSv9gviB60YXZ2TOsWEouucJmBM0c,1724
 ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB82-Pq9o1BvCinKE_WPvVxPTEcZjlJbFYM,2371
 ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
 ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json,sha256=C3uwkibJ08C9sOX-54kulZYmJlIpZ-SQpfE6HNGrjbM,343502
-ragaai_catalyst/tracers/utils/trace_json_converter.py,sha256=RH9dkCnPaSygvPvAilRE4lUdUaRtALJKH85E4jHcVoM,14072
+ragaai_catalyst/tracers/utils/rag_trace_json_converter.py,sha256=XSanJ3ChRn_Nqj5mlJhy-jFUCAhkw6lBuhhzUx6Mo9k,11239
+ragaai_catalyst/tracers/utils/trace_json_converter.py,sha256=E0_QfciQMMpCtQYrNB4l8HJhlaFalr5bkMqkVRgQahY,14073
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.6.4b0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.6.4b0.dist-info/METADATA,sha256=MZYKIXnjqIFIMsBq3edB7ayphprmPEozDDyjtNk8Lkc,22141
-ragaai_catalyst-2.1.6.4b0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ragaai_catalyst-2.1.6.4b0.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.6.4b0.dist-info/RECORD,,
+ragaai_catalyst-2.1.7.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.7.dist-info/METADATA,sha256=_4zHpKGxANGDv7HlC-pqZ6mZBqdvo4b8vDHyeKaefBY,22137
+ragaai_catalyst-2.1.7.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ragaai_catalyst-2.1.7.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.7.dist-info/RECORD,,