ragaai-catalyst 2.1.5b27__py3-none-any.whl → 2.1.5b29__py3-none-any.whl

ragaai_catalyst/dataset.py

@@ -696,6 +696,36 @@ class Dataset:
         except (IOError, UnicodeError) as e:
             logger.error(f"Error converting JSONL to CSV: {e}")
             raise
+        finally:
+            if os.path.exists(tmp_csv_path):
+                try:
+                    os.remove(tmp_csv_path)
+                except Exception as e:
+                    logger.error(f"Error removing temporary CSV file: {e}")
+
+    def create_from_df(self, df, dataset_name, schema_mapping):
+        tmp_csv_path = os.path.join(tempfile.gettempdir(), f"{dataset_name}.csv")
+        try:
+            df.to_csv(tmp_csv_path, index=False)
+            self.create_from_csv(tmp_csv_path, dataset_name, schema_mapping)
+        except (IOError, UnicodeError) as e:
+            logger.error(f"Error converting DataFrame to CSV: {e}")
+            raise
+        finally:
+            if os.path.exists(tmp_csv_path):
+                try:
+                    os.remove(tmp_csv_path)
+                except Exception as e:
+                    logger.error(f"Error removing temporary CSV file: {e}")
+
+    def add_rows_from_df(self, df, dataset_name):
+        tmp_csv_path = os.path.join(tempfile.gettempdir(), f"{dataset_name}.csv")
+        try:
+            df.to_csv(tmp_csv_path, index=False)
+            self.add_rows(tmp_csv_path, dataset_name)
+        except (IOError, UnicodeError) as e:
+            logger.error(f"Error converting DataFrame to CSV: {e}")
+            raise
         finally:
             if os.path.exists(tmp_csv_path):
                 try:

ragaai_catalyst/synthetic_data_generation.py

@@ -84,7 +84,6 @@ class SyntheticDataGeneration:
                 
             try:
                 system_message = self._get_system_message(question_type, current_batch_size)
-                
                 if "internal_llm_proxy" in kwargs:
                     batch_df = self._generate_internal_response(text, system_message, model_config, kwargs)
                 else:
@@ -232,7 +231,8 @@ class SyntheticDataGeneration:
             ValueError: If an invalid question type is specified.
         """
         if question_type == 'simple':
-            return f'''Generate a set of {n} very simple questions answerable in a single phrase. 
+            return f'''Generate a set of {n} very simple questions answerable in a single phrase using the below text.
+                Only generate questions answerable from the text given, to cover all parts of the given document. 
                 Also return the answers for the generated questions.
                 Return the response in a list of object format. 
                 Each object in list should have Question and corresponding answer.
@@ -241,6 +241,7 @@ class SyntheticDataGeneration:
             '''
         elif question_type == 'mcq':
             return f'''Generate a set of {n} questions with 4 probable answers from the given text. 
+                Only generate questions answerable from the text given, to cover all parts of the given document. 
                 The options should not be longer than a phrase. There should be only 1 correct answer.
                 There should not be any ambiguity between correct and incorrect options.
                 Return the response in a list of object format. 
@@ -250,6 +251,7 @@ class SyntheticDataGeneration:
             '''
         elif question_type == 'complex':
             return f'''Can you generate a set of {n} complex questions answerable in long form from the below texts.
+                Only generate questions answerable from the text given, to cover all parts of the given document. 
                 Make sure the questions are important and provide new information to the user.
                 Return the response in a list of object format. Enclose any quotes in single quote. 
                 Do not use double quotes within questions or answers.

{ragaai_catalyst-2.1.5b27.dist-info → ragaai_catalyst-2.1.5b29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ragaai_catalyst
-Version: 2.1.5b27
+Version: 2.1.5b29
 Summary: RAGA AI CATALYST
 Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
 Requires-Python: <3.13,>=3.9
@@ -51,10 +51,11 @@ RagaAI Catalyst is a comprehensive platform designed to enhance the management a
 
 ![RagaAI Catalyst](docs/img/main.png)
 
+![RagaAI Catalyst](docs/img/main.png)
+
 ## Table of Contents
 
 - [RagaAI Catalyst](#ragaai-catalyst)
-  - [Table of Contents](#table-of-contents)
   - [Installation](#installation)
   - [Configuration](#configuration)
   - [Usage](#usage)
@@ -62,10 +63,10 @@ RagaAI Catalyst is a comprehensive platform designed to enhance the management a
     - [Dataset Management](#dataset-management)
     - [Evaluation Management](#evaluation)
     - [Trace Management](#trace-management)
+    - [Agentic Tracing](#agentic-tracing)
     - [Prompt Management](#prompt-management)
     - [Synthetic Data Generation](#synthetic-data-generation)
     - [Guardrail Management](#guardrail-management)
-    - [Agentic Tracing](#agentic-tracing)
     - [Red-teaming](#red-teaming)
 
 ## Installation
@@ -89,7 +90,15 @@ catalyst = RagaAICatalyst(
     base_url="BASE_URL"
 )
 ```
-**Note**: Authetication to RagaAICatalyst is necessary to perform any operations below 
+you'll need to generate authentication credentials:
+
+1. Navigate to your profile settings
+2. Select "Authenticate" 
+3. Click "Generate New Key" to create your access and secret keys
+
+![How to generate authentication keys](docs/img/autheticate.gif)
+
+**Note**: Authetication to RagaAICatalyst is necessary to perform any operations below.
 
 
 ## Usage
@@ -112,7 +121,7 @@ catalyst.project_use_cases()
 projects = catalyst.list_projects()
 print(projects)
 ```
-![Projects](docs/img/projects_new.png)
+![Projects](docs/img/create_project.gif)
 
 ### Dataset Management
 Manage datasets efficiently for your projects:
@@ -134,11 +143,25 @@ dataset_manager.create_from_csv(
     schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
 )
 
+# Create a dataset from JSONl
+dataset_manager.create_from_jsonl(
+    jsonl_path='jsonl_path',
+    dataset_name='MyDataset',
+    schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
+)
+
+# Create a dataset from dataframe
+dataset_manager.create_from_df(
+    df=df,
+    dataset_name='MyDataset',
+    schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
+)
+
 # Get project schema mapping
 dataset_manager.get_schema_mapping()
 
 ```
-![Dataset](docs/img/dataset.png)
+![Dataset](docs/img/dataset.gif)
 
 For more detailed information on Dataset Management, including CSV schema handling and advanced usage, please refer to the [Dataset Management documentation](docs/dataset_management.md).
 
@@ -192,32 +215,48 @@ print("Experiment Status:", status)
 # Get the results of the experiment
 results = evaluation.get_results()
 print("Experiment Results:", results)
+
+# Appending Metrics for New Data
+# If you've added new rows to your dataset, you can calculate metrics just for the new data:
+evaluation.append_metrics(display_name="Faithfulness_v1")
 ```
-![Evaluation](docs/img/evaluation.png)
+
+![Evaluation](docs/img/evaluation.gif)
+
 
 
 ### Trace Management
 
 Record and analyze traces of your RAG application:
-
+            
 ```python
-from ragaai_catalyst import Tracer
+from ragaai_catalyst import RagaAICatalyst, Tracer
 
-# Start a trace recording
 tracer = Tracer(
     project_name="Test-RAG-App-1",
     dataset_name="tracer_dataset_name",
-    metadata={"key1": "value1", "key2": "value2"},
-    tracer_type="langchain",
-    pipeline={
-        "llm_model": "gpt-4o-mini",
-        "vector_store": "faiss",
-        "embed_model": "text-embedding-ada-002",
-    }
-).start()
+    tracer_type="tracer_type"
+)
+```
 
-# Your code here
+There are two ways to start a trace recording
+
+1- with tracer():
+
+```python
 
+with tracer():
+    # Your code here
+
+```
+
+2- tracer.start()
+
+```python
+#start the trace recording
+tracer.start()
+
+# Your code here
 
 # Stop the trace recording
 tracer.stop()
@@ -225,8 +264,192 @@ tracer.stop()
 # Get upload status
 tracer.get_upload_status()
 ```
+
 ![Trace](docs/img/trace_comp.png)
+For more detailed information on Trace Management, please refer to the [Trace Management documentation](docs/trace_management.md).
 
+### Agentic Tracing
+
+The Agentic Tracing module provides comprehensive monitoring and analysis capabilities for AI agent systems. It helps track various aspects of agent behavior including:
+
+- LLM interactions and token usage
+- Tool utilization and execution patterns
+- Network activities and API calls
+- User interactions and feedback
+- Agent decision-making processes
+
+The module includes utilities for cost tracking, performance monitoring, and debugging agent behavior. This helps in understanding and optimizing AI agent performance while maintaining transparency in agent operations.
+
+#### Tracer initialization
+
+Initialize the tracer with project_name and dataset_name
+
+```python
+from ragaai_catalyst import RagaAICatalyst, Tracer, trace_llm, trace_tool, trace_agent, current_span
+
+agentic_tracing_dataset_name = "agentic_tracing_dataset_name"
+
+tracer = Tracer(
+    project_name=agentic_tracing_project_name,
+    dataset_name=agentic_tracing_dataset_name,
+    tracer_type="Agentic",
+)
+```
+
+```python
+from ragaai_catalyst import init_tracing
+init_tracing(catalyst=catalyst, tracer=tracer)
+```
+
+#### Agentic Tracing Features
+1- add span level metrics
+
+```python
+current_span().add_metrics(name='Accuracy', score=0.5, reasoning='some reasoning')
+```
+
+2- add trace level metrics
+
+```python
+tracer.add_metrics(name='hallucination_1', score=0.5, reasoning='some reasoning')
+```
+
+3- add gt 
+
+```python
+current_span().add_gt("This is the ground truth")
+```
+
+4- add context
+
+```python
+current_span().add_context("This is the context")
+```
+
+5- add span level metric execution
+
+```python
+current_span().execute_metrics(
+    name="Hallucination",
+    model="gpt-4o",
+    provider="openai"
+)
+```
+
+#### Example
+```python
+from ragaai_catalyst import trace_llm, trace_tool, trace_agent, current_span
+
+from openai import OpenAI
+
+
+@trace_llm(name="llm_call", tags=["default_llm_call"])
+def llm_call(prompt, max_tokens=512, model="gpt-4o-mini"):
+    client = OpenAI(api_key=OPENAI_API_KEY)
+    response = client.chat.completions.create(
+        model=model,
+        messages=[{"role": "user", "content": prompt}],
+        max_tokens=max_tokens,
+        temperature=0.85,
+    )
+    # Span level context
+    current_span().add_context("name = span level in summary_agent, context = some span level context")
+
+    # Span level execute metrics
+    current_span().execute_metrics(
+        name="Hallucination",
+        model="gpt-4o",
+        provider="openai"
+    )
+    response_data = response.choices[0].message.content.strip()
+    print('response_data: ', response_data)
+    return response_data
+
+class SummaryAgent:
+    def __init__(self, persona="Summary Agent"):
+        self.persona = persona
+
+    @trace_agent(name="summary_agent")
+    def summarize(self, text):
+        prompt = f"Please summarize this text concisely: {text}"
+
+        # Span level metric
+        current_span().add_metrics(name='Accuracy', score=0.5, reasoning='some reasoning')
+
+        # Span level context
+        current_span().add_context("name = span level in summary_agent, context = some span level context")
+
+        summary = llm_call(prompt)
+        return summary
+
+
+class AnalysisAgent:
+    def __init__(self, persona="Analysis Agent"):
+        self.persona = persona
+        self.summary_agent = SummaryAgent()
+
+    @trace_agent(name="analysis_agent")
+    def analyze(self, text):
+        summary = self.summary_agent.summarize(text)
+
+        prompt = f"Given this summary: {summary}\nProvide a brief analysis of the main points."
+
+        # Span level metric
+        current_span().add_metrics(name='correctness', score=0.5, reasoning='some reasoning')
+        analysis = llm_call(prompt)
+
+        return {
+            "summary": summary,
+            "analysis": analysis
+        }
+
+class RecommendationAgent:
+    def __init__(self, persona="Recommendation Agent"):
+        self.persona = persona
+        self.analysis_agent = AnalysisAgent()
+
+    @trace_agent(name="recommendation_agent", tags=['coordinator_agent'])
+    def recommend(self, text):
+        analysis_result = self.analysis_agent.analyze(text)
+
+        prompt = f"""Given this summary: {analysis_result['summary']}
+        And this analysis: {analysis_result['analysis']}
+        Provide 2-3 actionable recommendations."""
+
+        recommendations = llm_call(prompt)
+
+        return {
+            "summary": analysis_result["summary"],
+            "analysis": analysis_result["analysis"],
+            "recommendations": recommendations
+        }
+#Defining agent tracer
+@trace_agent(name="get_recommendation", tags=['coordinator_agent'])
+def get_recommendation(agent, text):
+    recommendation = agent.recommend(text)
+    return recommendation
+
+def main():
+    text = """
+    Artificial Intelligence has transformed various industries in recent years.
+    From healthcare to finance, AI applications are becoming increasingly prevalent.
+    Machine learning models are being used to predict market trends, diagnose diseases,
+    and automate routine tasks. The impact of AI on society continues to grow,
+    raising both opportunities and challenges for the future.
+    """
+
+    recommendation_agent = RecommendationAgent()
+    result = get_recommendation(recommendation_agent, text)
+
+
+    # Trace level metric
+    tracer.add_metrics(name='hallucination_1', score=0.5, reasoning='some reasoning')
+
+# Run tracer
+with tracer:
+    main()
+```
+![Tracing](docs/img/last_main.png)
 
 ### Prompt Management
 
@@ -401,45 +624,6 @@ executor([message],prompt_params,model_params,llm_caller)
 ```
 ![Guardrails](docs/img/guardrails.png)
 
-### Agentic Tracing
-
-The Agentic Tracing module provides comprehensive monitoring and analysis capabilities for AI agent systems. It helps track various aspects of agent behavior including:
-
-- LLM interactions and token usage
-- Tool utilization and execution patterns
-- Network activities and API calls
-- User interactions and feedback
-- Agent decision-making processes
-
-The module includes utilities for cost tracking, performance monitoring, and debugging agent behavior. This helps in understanding and optimizing AI agent performance while maintaining transparency in agent operations.
-
-```python
-from ragaai_catalyst import AgenticTracer
-
-# Initialize tracer
-tracer = AgenticTracer(
-    project_name="project_name",
-    dataset_name="dataset_name",
-    tracer_type="agentic",
-)
-
-# Define tracers
-@tracer.trace_agents("agent_name")
-# Agent Definition
-
-@tracer.trace_llm("llm_name")
-# LLM Definition
-
-@tracer.trace_tool("tool_name")
-# Tool Definition
-
-# Perform tracing
-with tracer:
-    # Agent execution code
-    pass
-```
-![Tracing](docs/img/last_main.png)
-
 ### Red-teaming
 
 The Red-teaming module provides comprehensive scans for model vulnerabilities:

{ragaai_catalyst-2.1.5b27.dist-info → ragaai_catalyst-2.1.5b29.dist-info}/RECORD

@@ -1,6 +1,6 @@
 ragaai_catalyst/__init__.py,sha256=xGqvQoS_Ir_Lup1YUIVc5VlsIplRMnnh_-6qK_eB0u4,843
 ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
-ragaai_catalyst/dataset.py,sha256=u4QofzdH1_bGsZ-AFc-qFMGq9K-H-YZHqmLSFG8AEDI,28120
+ragaai_catalyst/dataset.py,sha256=YCj8Ovu6y38KEw-1HCe4xQWkmYPgfNTtMa8Q0g6B62o,29401
 ragaai_catalyst/evaluation.py,sha256=O96CydYVPh3duUmXjY6REIXMOR-tOPixSG-Qhrf636A,22955
 ragaai_catalyst/experiment.py,sha256=8yQo1phCHlpnJ-4CqCaIbLXg_1ZlAuLGI9kqGBl-OTE,18859
 ragaai_catalyst/guard_executor.py,sha256=llPbE3DyVtrybojXknzBZj8-dtUrGBQwi9-ZiPJxGRo,3762
@@ -10,7 +10,7 @@ ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKs
 ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
 ragaai_catalyst/ragaai_catalyst.py,sha256=5nVg3_-lcvhrXjNkPTeGhe3tdUjm_4ZIctOcqWXBkRA,17939
 ragaai_catalyst/redteaming.py,sha256=pvHfwaHZgrq0HYhygEUm6-WotAxA2X9Xg1Kj9NlEzAI,6803
-ragaai_catalyst/synthetic_data_generation.py,sha256=etqG0AHzC0V1B5fTAOEJxOJ9lhWZyNVmwC9DvTDA-gs,21269
+ragaai_catalyst/synthetic_data_generation.py,sha256=QdabyUErLHhEwE-grlD-OJgqbL9N0S_tCPp9GmuMSyo,21617
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
 ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
 ragaai_catalyst/tracers/distributed.py,sha256=MwlBwIxCAng-OI-7Ove_rkE1mTLeuW4Jw-wWEVJBNlI,9968
@@ -71,8 +71,8 @@ ragaai_catalyst/tracers/utils/convert_llama_instru_callback.py,sha256=8qLo7x4Zsn
 ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB82-Pq9o1BvCinKE_WPvVxPTEcZjlJbFYM,2371
 ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.5b27.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ragaai_catalyst-2.1.5b27.dist-info/METADATA,sha256=cFqgoEcNn3EKitmc2APLO__ZomawZQc_MEm0bflrHJ8,14160
-ragaai_catalyst-2.1.5b27.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.5b27.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.5b27.dist-info/RECORD,,
+ragaai_catalyst-2.1.5b29.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ragaai_catalyst-2.1.5b29.dist-info/METADATA,sha256=VtRy6oG0L-rNaky2LVNVNvXV6pAHjdRs6aMah4Qp3yM,19472
+ragaai_catalyst-2.1.5b29.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ragaai_catalyst-2.1.5b29.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.5b29.dist-info/RECORD,,