ragaai-catalyst 2.1.3__py3-none-any.whl → 2.1.4b0__py3-none-any.whl

ragaai_catalyst/tracers/tracer.py

@@ -191,6 +191,9 @@ class Tracer(AgenticTracing):
             print("Stopping tracer and initiating trace upload...")
             self._cleanup()
             self._upload_task = self._run_async(self._upload_traces())
+            self.is_active = False
+            self.dataset_name = None
+            
             return "Trace upload initiated. Use get_upload_status() to check the status."
         elif self.tracer_type == "llamaindex":
             from ragaai_catalyst.tracers.llamaindex_callback import LlamaIndexTracer

ragaai_catalyst-2.1.4b0.dist-info/METADATA

@@ -0,0 +1,430 @@
+Metadata-Version: 2.2
+Name: ragaai_catalyst
+Version: 2.1.4b0
+Summary: RAGA AI CATALYST
+Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
+Requires-Python: <3.13,>=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.10.2
+Requires-Dist: opentelemetry-api==1.25.0
+Requires-Dist: opentelemetry-sdk==1.25.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.25.0
+Requires-Dist: opentelemetry-instrumentation==0.46b0
+Requires-Dist: opentelemetry-instrumentation-fastapi==0.46b0
+Requires-Dist: opentelemetry-instrumentation-asgi==0.46b0
+Requires-Dist: opentelemetry-semantic-conventions==0.46b0
+Requires-Dist: opentelemetry-util-http==0.46b0
+Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
+Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
+Requires-Dist: langchain-core>=0.2.11
+Requires-Dist: langchain>=0.2.11
+Requires-Dist: openai>=1.57.0
+Requires-Dist: pandas
+Requires-Dist: groq>=0.11.0
+Requires-Dist: PyPDF2>=3.0.1
+Requires-Dist: google-generativeai>=0.8.2
+Requires-Dist: Markdown>=3.7
+Requires-Dist: litellm==1.51.1
+Requires-Dist: tenacity==8.3.0
+Requires-Dist: tqdm>=4.66.5
+Requires-Dist: llama-index<0.11.0,>=0.10.0
+Requires-Dist: pyopenssl>=24.2.1
+Requires-Dist: psutil~=6.0.0
+Requires-Dist: py-cpuinfo~=9.0.0
+Requires-Dist: requests~=2.32.3
+Requires-Dist: GPUtil~=1.4.0
+Requires-Dist: astor>=0.8.1
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+
+# RagaAI Catalyst&nbsp; ![GitHub release (latest by date)](https://img.shields.io/github/v/release/raga-ai-hub/ragaai-catalyst) ![GitHub stars](https://img.shields.io/github/stars/raga-ai-hub/ragaai-catalyst?style=social)  ![Issues](https://img.shields.io/github/issues/raga-ai-hub/ragaai-catalyst) 
+
+RagaAI Catalyst is a comprehensive platform designed to enhance the management and optimization of LLM projects. It offers a wide range of features, including project management, dataset management, evaluation management, trace management, prompt management, synthetic data generation, and guardrail management. These functionalities enable you to efficiently evaluate, and safeguard your LLM applications.
+
+## Table of Contents
+
+- [RagaAI Catalyst](#ragaai-catalyst)
+  - [Table of Contents](#table-of-contents)
+  - [Installation](#installation)
+  - [Configuration](#configuration)
+  - [Usage](#usage)
+    - [Project Management](#project-management)
+    - [Dataset Management](#dataset-management)
+    - [Evaluation Management](#evaluation)
+    - [Trace Management](#trace-management)
+    - [Prompt Management](#prompt-management)
+    - [Synthetic Data Generation](#synthetic-data-generation)
+    - [Guardrail Management](#guardrail-management)
+    - [Agentic Tracing](#agentic-tracing)
+
+## Installation
+
+To install RagaAI Catalyst, you can use pip:
+
+```bash
+pip install ragaai-catalyst
+```
+
+## Configuration
+
+Before using RagaAI Catalyst, you need to set up your credentials. You can do this by setting environment variables or passing them directly to the `RagaAICatalyst` class:
+
+```python
+from ragaai_catalyst import RagaAICatalyst
+
+catalyst = RagaAICatalyst(
+    access_key="YOUR_ACCESS_KEY",
+    secret_key="YOUR_SECRET_KEY",
+    base_url="BASE_URL"
+)
+```
+**Note**: Authetication to RagaAICatalyst is necessary to perform any operations below 
+
+
+## Usage
+
+### Project Management
+
+Create and manage projects using RagaAI Catalyst:
+
+```python
+# Create a project
+project = catalyst.create_project(
+    project_name="Test-RAG-App-1",
+    usecase="Chatbot"
+)
+
+# Get project usecases
+catalyst.project_use_cases()
+
+# List projects
+projects = catalyst.list_projects()
+print(projects)
+```
+
+### Dataset Management
+Manage datasets efficiently for your projects:
+
+```py
+from ragaai_catalyst import Dataset
+
+# Initialize Dataset management for a specific project
+dataset_manager = Dataset(project_name="project_name")
+
+# List existing datasets
+datasets = dataset_manager.list_datasets()
+print("Existing Datasets:", datasets)
+
+# Create a dataset from CSV
+dataset_manager.create_from_csv(
+    csv_path='path/to/your.csv',
+    dataset_name='MyDataset',
+    schema_mapping={'column1': 'schema_element1', 'column2': 'schema_element2'}
+)
+
+# Get project schema mapping
+dataset_manager.get_schema_mapping()
+
+```
+
+For more detailed information on Dataset Management, including CSV schema handling and advanced usage, please refer to the [Dataset Management documentation](docs/dataset_management.md).
+
+
+### Evaluation
+
+Create and manage metric evaluation of your RAG application:
+
+```python
+from ragaai_catalyst import Evaluation
+
+# Create an experiment
+evaluation = Evaluation(
+    project_name="Test-RAG-App-1",
+    dataset_name="MyDataset",
+)
+
+# Get list of available metrics
+evaluation.list_metrics()
+
+# Add metrics to the experiment
+
+schema_mapping={
+    'Query': 'prompt',
+    'response': 'response',
+    'Context': 'context',
+    'expectedResponse': 'expected_response'
+}
+
+# Add single metric
+evaluation.add_metrics(
+    metrics=[
+      {"name": "Faithfulness", "config": {"model": "gpt-4o-mini", "provider": "openai", "threshold": {"gte": 0.232323}}, "column_name": "Faithfulness_v1", "schema_mapping": schema_mapping},
+    
+    ]
+)
+
+# Add multiple metrics
+evaluation.add_metrics(
+    metrics=[
+        {"name": "Faithfulness", "config": {"model": "gpt-4o-mini", "provider": "openai", "threshold": {"gte": 0.323}}, "column_name": "Faithfulness_gte", "schema_mapping": schema_mapping},
+        {"name": "Hallucination", "config": {"model": "gpt-4o-mini", "provider": "openai", "threshold": {"lte": 0.323}}, "column_name": "Hallucination_lte", "schema_mapping": schema_mapping},
+        {"name": "Hallucination", "config": {"model": "gpt-4o-mini", "provider": "openai", "threshold": {"eq": 0.323}}, "column_name": "Hallucination_eq", "schema_mapping": schema_mapping},
+    ]
+)
+
+# Get the status of the experiment
+status = evaluation.get_status()
+print("Experiment Status:", status)
+
+# Get the results of the experiment
+results = evaluation.get_results()
+print("Experiment Results:", results)
+```
+
+
+
+### Trace Management
+
+Record and analyze traces of your RAG application:
+
+```python
+from ragaai_catalyst import Tracer
+
+# Start a trace recording
+tracer = Tracer(
+    project_name="Test-RAG-App-1",
+    dataset_name="tracer_dataset_name",
+    metadata={"key1": "value1", "key2": "value2"},
+    tracer_type="langchain",
+    pipeline={
+        "llm_model": "gpt-4o-mini",
+        "vector_store": "faiss",
+        "embed_model": "text-embedding-ada-002",
+    }
+).start()
+
+# Your code here
+
+
+# Stop the trace recording
+tracer.stop()
+
+# Get upload status
+tracer.get_upload_status()
+```
+
+
+### Prompt Management
+
+Manage and use prompts efficiently in your projects:
+
+```py
+from ragaai_catalyst import PromptManager
+
+# Initialize PromptManager
+prompt_manager = PromptManager(project_name="Test-RAG-App-1")
+
+# List available prompts
+prompts = prompt_manager.list_prompts()
+print("Available prompts:", prompts)
+
+# Get default prompt by prompt_name
+prompt_name = "your_prompt_name"
+prompt = prompt_manager.get_prompt(prompt_name)
+
+# Get specific version of prompt by prompt_name and version
+prompt_name = "your_prompt_name"
+version = "v1"
+prompt = prompt_manager.get_prompt(prompt_name,version)
+
+# Get variables in a prompt
+variable = prompt.get_variables()
+print("variable:",variable)
+
+# Get prompt content
+prompt_content = prompt.get_prompt_content()
+print("prompt_content:", prompt_content)
+
+# Compile the prompt with variables
+compiled_prompt = prompt.compile(query="What's the weather?", context="sunny", llm_response="It's sunny today")
+print("Compiled prompt:", compiled_prompt)
+
+# implement compiled_prompt with openai
+import openai
+def get_openai_response(prompt):
+    client = openai.OpenAI()
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=prompt
+    )
+    return response.choices[0].message.content
+openai_response = get_openai_response(compiled_prompt)
+print("openai_response:", openai_response)
+
+# implement compiled_prompt with litellm
+import litellm
+def get_litellm_response(prompt):
+    response = litellm.completion(
+        model="gpt-4o-mini",
+        messages=prompt
+    )
+    return response.choices[0].message.content
+litellm_response = get_litellm_response(compiled_prompt)
+print("litellm_response:", litellm_response)
+
+```
+For more detailed information on Prompt Management, please refer to the [Prompt Management documentation](docs/prompt_management.md).
+
+
+### Synthetic Data Generation
+
+```py
+from ragaai_catalyst import SyntheticDataGeneration
+
+# Initialize Synthetic Data Generation
+sdg = SyntheticDataGeneration()
+
+# Process your file
+text = sdg.process_document(input_data="file_path")
+
+# Generate results
+result = sdg.generate_qna(text, question_type ='complex',model_config={"provider":"openai","model":"openai/gpt-3.5-turbo"},n=5)
+
+print(result.head())
+
+# Get supported Q&A types
+sdg.get_supported_qna()
+
+# Get supported providers
+sdg.get_supported_providers()
+```
+
+
+
+### Guardrail Management
+
+```py
+from ragaai_catalyst import GuardrailsManager
+
+# Initialize Guardrails Manager
+gdm = GuardrailsManager(project_name=project_name)
+
+# Get list of Guardrails available
+guardrails_list = gdm.list_guardrails()
+print('guardrails_list:', guardrails_list)
+
+# Get list of fail condition for guardrails
+fail_conditions = gdm.list_fail_condition()
+print('fail_conditions;', fail_conditions)
+
+#Get list of deployment ids
+deployment_list = gdm.list_deployment_ids()
+print('deployment_list:', deployment_list)
+
+# Get specific deployment id with guardrails information
+deployment_id_detail = gdm.get_deployment(17)
+print('deployment_id_detail:', deployment_id_detail)
+
+# Add guardrails to a deployment id
+guardrails_config = {"guardrailFailConditions": ["FAIL"],
+                     "deploymentFailCondition": "ALL_FAIL",
+                     "alternateResponse": "Your alternate response"}
+
+guardrails = [
+    {
+      "displayName": "Response_Evaluator",
+      "name": "Response Evaluator",
+      "config":{
+          "mappings": [{
+                        "schemaName": "Text",
+                        "variableName": "Response"
+                    }],
+          "params": {
+                    "isActive": {"value": False},
+                    "isHighRisk": {"value": True},
+                    "threshold": {"eq": 0},
+                    "competitors": {"value": ["Google","Amazon"]}
+                }
+      }
+    },
+    {
+      "displayName": "Regex_Check",
+      "name": "Regex Check",
+      "config":{
+          "mappings": [{
+                        "schemaName": "Text",
+                        "variableName": "Response"
+                    }],
+          "params":{
+              "isActive": {"value": False},
+              "isHighRisk": {"value": True},
+              "threshold": {"lt1": 1}
+          }
+      }
+    }
+]
+
+gdm.add_guardrails(deployment_id, guardrails, guardrails_config)
+
+
+# Import GuardExecutor
+from ragaai_catalyst import GuardExecutor
+
+# Initialise GuardExecutor with required params and Evaluate
+executor = GuardExecutor(deployment_id,gdm,field_map={'context':'document'})
+
+
+message={'role':'user',
+         'content':'What is the capital of France'
+        }
+prompt_params={'document':' France'}
+
+model_params = {'temperature':.7,'model':'gpt-4o-mini'}
+llm_caller = 'litellm'
+
+executor([message],prompt_params,model_params,llm_caller)
+
+```
+
+### Agentic Tracing
+
+The Agentic Tracing module provides comprehensive monitoring and analysis capabilities for AI agent systems. It helps track various aspects of agent behavior including:
+
+- LLM interactions and token usage
+- Tool utilization and execution patterns
+- Network activities and API calls
+- User interactions and feedback
+- Agent decision-making processes
+
+The module includes utilities for cost tracking, performance monitoring, and debugging agent behavior. This helps in understanding and optimizing AI agent performance while maintaining transparency in agent operations.
+
+```python
+from ragaai_catalyst import AgenticTracer
+
+# Initialize tracer
+tracer = AgenticTracer(
+    project_name="project_name",
+    dataset_name="dataset_name",
+    tracer_type="agentic",
+)
+
+# Define tracers
+@tracer.trace_agents("agent_name")
+# Agent Definition
+
+@tracer.trace_llm("llm_name")
+# LLM Definition
+
+@tracer.trace_tool("tool_name")
+# Tool Definition
+
+# Perform tracing
+with tracer:
+    # Agent execution code
+    pass
+

{ragaai_catalyst-2.1.3.dist-info → ragaai_catalyst-2.1.4b0.dist-info}/RECORD

@@ -13,7 +13,7 @@ ragaai_catalyst/synthetic_data_generation.py,sha256=uDV9tNwto2xSkWg5XHXUvjErW-4P
 ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
 ragaai_catalyst/tracers/__init__.py,sha256=yxepo7iVjTNI_wFdk3Z6Ghu64SazVyszCPEHYrX5WQk,50
 ragaai_catalyst/tracers/llamaindex_callback.py,sha256=vPE7MieKjfwLrLUnnPs20Df0xNYqoCCj-Mt2NbiuiKU,14023
-ragaai_catalyst/tracers/tracer.py,sha256=NgJrhC-nEbSYeFHYvWtzg73V1XokWVVoTs5z1n-xQUs,12716
+ragaai_catalyst/tracers/tracer.py,sha256=MPyrmJ1T68jOkbXx3WwnrzJjGbd54C_9ZhGDfu2tfZE,12801
 ragaai_catalyst/tracers/upload_traces.py,sha256=hs0PEmit3n3_uUqrdbwcBdyK5Nbkik3JQVwJMEwYTd4,4796
 ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
 ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
@@ -54,7 +54,7 @@ ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hak
 ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
 ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
 ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
-ragaai_catalyst-2.1.3.dist-info/METADATA,sha256=H6mSUPhVI2PggdxVms8ggrA_0Y0J3v3TzbRewfDj5Rw,1802
-ragaai_catalyst-2.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ragaai_catalyst-2.1.3.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
-ragaai_catalyst-2.1.3.dist-info/RECORD,,
+ragaai_catalyst-2.1.4b0.dist-info/METADATA,sha256=E3Uqt-UmOtCN4VKHgjt_1LA57p9woqutuWce1NDxXaU,12745
+ragaai_catalyst-2.1.4b0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ragaai_catalyst-2.1.4b0.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
+ragaai_catalyst-2.1.4b0.dist-info/RECORD,,

ragaai_catalyst-2.1.3.dist-info/METADATA

@@ -1,43 +0,0 @@
-Metadata-Version: 2.2
-Name: ragaai_catalyst
-Version: 2.1.3
-Summary: RAGA AI CATALYST
-Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
-Requires-Python: <3.13,>=3.9
-Description-Content-Type: text/markdown
-Requires-Dist: aiohttp>=3.10.2
-Requires-Dist: opentelemetry-api==1.25.0
-Requires-Dist: opentelemetry-sdk==1.25.0
-Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.25.0
-Requires-Dist: opentelemetry-instrumentation==0.46b0
-Requires-Dist: opentelemetry-instrumentation-fastapi==0.46b0
-Requires-Dist: opentelemetry-instrumentation-asgi==0.46b0
-Requires-Dist: opentelemetry-semantic-conventions==0.46b0
-Requires-Dist: opentelemetry-util-http==0.46b0
-Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
-Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
-Requires-Dist: langchain-core>=0.2.11
-Requires-Dist: langchain>=0.2.11
-Requires-Dist: openai>=1.57.0
-Requires-Dist: pandas
-Requires-Dist: groq>=0.11.0
-Requires-Dist: PyPDF2>=3.0.1
-Requires-Dist: google-generativeai>=0.8.2
-Requires-Dist: Markdown>=3.7
-Requires-Dist: litellm==1.51.1
-Requires-Dist: tenacity==8.3.0
-Requires-Dist: tqdm>=4.66.5
-Requires-Dist: llama-index<0.11.0,>=0.10.0
-Requires-Dist: pyopenssl>=24.2.1
-Requires-Dist: psutil~=6.0.0
-Requires-Dist: py-cpuinfo~=9.0.0
-Requires-Dist: requests~=2.32.3
-Requires-Dist: GPUtil~=1.4.0
-Requires-Dist: astor>=0.8.1
-Provides-Extra: dev
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: pytest-cov; extra == "dev"
-Requires-Dist: black; extra == "dev"
-Requires-Dist: isort; extra == "dev"
-Requires-Dist: mypy; extra == "dev"
-Requires-Dist: flake8; extra == "dev"