ragas-examples 0.3.6__tar.gz → 0.3.7__tar.gz

{ragas_examples-0.3.6/ragas_examples.egg-info → ragas_examples-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ragas-examples
-Version: 0.3.6
+Version: 0.3.7
 Summary: Official examples for the ragas project
 Author: Ragas Team
 License: Apache-2.0

{ragas_examples-0.3.6 → ragas_examples-0.3.7}/iterate_prompt/evals.py

@@ -69,7 +69,7 @@ def priority_accuracy(prediction: str, expected_priority: str):
 async def support_triage_experiment(row, prompt_file: str, experiment_name: str):
     """Experiment function for support triage evaluation."""
     # Get model response
-    response = await asyncio.to_thread(run_prompt, row["text"], prompt_file=prompt_file)
+    response = await run_prompt(row["text"], prompt_file=prompt_file)
 
     # Parse response to extract predicted values
     try:

{ragas_examples-0.3.6 → ragas_examples-0.3.7}/iterate_prompt/run_prompt.py

@@ -1,8 +1,8 @@
 import os
 
-from openai import OpenAI
+from openai import AsyncOpenAI
 
-client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+client = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
 
 def load_prompt(prompt_file: str) -> str:
@@ -11,12 +11,12 @@ def load_prompt(prompt_file: str) -> str:
         return f.read().strip()
 
 
-def run_prompt(ticket_text: str, prompt_file: str = "promptv1.txt"):
+async def run_prompt(ticket_text: str, prompt_file: str = "promptv1.txt"):
     """Run the prompt against a customer support ticket"""
     system_prompt = load_prompt(prompt_file)
     user_message = f'Ticket: "{ticket_text}"'
 
-    response = client.chat.completions.create(
+    response = await client.chat.completions.create(
         model="gpt-5-mini-2025-08-07",
         response_format={"type": "json_object"},
         messages=[
@@ -33,9 +33,10 @@ def run_prompt(ticket_text: str, prompt_file: str = "promptv1.txt"):
 
 
 if __name__ == "__main__":
+    import asyncio
     # Test with a sample customer support ticket
     test_ticket = "SSO via Okta succeeds then bounces me back to /login with no session. Colleagues can sign in. I tried clearing cookies; same result. Error in devtools: state mismatch. I'm blocked from our boards."
     print("Test ticket:")
     print(f'"{test_ticket}"')
     print("\nResponse:")
-    print(run_prompt(test_ticket))
+    print(asyncio.run(run_prompt(test_ticket)))

{ragas_examples-0.3.6 → ragas_examples-0.3.7}/ragas_examples/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.6'
-__version_tuple__ = version_tuple = (0, 3, 6)
+__version__ = version = '0.3.7'
+__version_tuple__ = version_tuple = (0, 3, 7)
 
-__commit_id__ = commit_id = 'g49f47f18e'
+__commit_id__ = commit_id = 'gf31c365e9'

{ragas_examples-0.3.6 → ragas_examples-0.3.7}/ragas_examples/benchmark_llm/evals.py

@@ -1,5 +1,4 @@
 import argparse
-import asyncio
 import datetime
 import json
 import os
@@ -7,6 +6,10 @@ import sys
 from typing import List, Optional
 
 import pandas as pd
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv(".env")
 
 from ragas import experiment
 from ragas.dataset import Dataset
@@ -35,50 +38,44 @@ def discount_accuracy(prediction: str, expected_discount):
         )
 
 
-def create_benchmark_experiment(model_name: str, experiment_name: str):
-    """Factory function to create experiment functions for different models."""
-
-    @experiment()
-    async def benchmark_experiment(row):
-        # Get model response (run blocking call in thread to keep async runner responsive)
-        response = await asyncio.to_thread(
-            run_prompt, row["customer_profile"], model=model_name
-        )
-
-        # Parse response (strict JSON mode expected)
-        try:
-            parsed_json = json.loads(response)
-            predicted_discount = parsed_json.get("discount_percentage")
-        except Exception:
-            predicted_discount = None
+@experiment()
+async def benchmark_experiment(row, model_name: str):
+    """Benchmark experiment function that evaluates a model on discount calculation."""
+    # Get model response
+    response = await run_prompt(row["customer_profile"], model=model_name)
 
-        # Score the response
-        score = discount_accuracy.score(
-            prediction=response, expected_discount=row["expected_discount"]
-        )
+    # Parse response (strict JSON mode expected)
+    try:
+        parsed_json = json.loads(response)
+        predicted_discount = parsed_json.get("discount_percentage")
+    except Exception:
+        predicted_discount = None
 
-        return {
-            **row,
-            "model": model_name,
-            "experiment_name": experiment_name,
-            "response": response,
-            "predicted_discount": predicted_discount,
-            "score": score.value,
-            "score_reason": score.reason,
-        }
+    # Score the response
+    score = discount_accuracy.score(
+        prediction=response, expected_discount=row["expected_discount"]
+    )
 
-    return benchmark_experiment
+    return {
+        **row,
+        "model": model_name,
+        "response": response,
+        "predicted_discount": predicted_discount,
+        "score": score.value,
+        "score_reason": score.reason,
+    }
 
 
 def load_dataset():
-    """Load the dataset from CSV file."""
-    # Get the directory where this file is located
+    """Load the dataset from CSV file. Downloads from GitHub if not found locally."""
+    import urllib.request
     current_dir = os.path.dirname(os.path.abspath(__file__))
-
-    dataset = Dataset.load(
-        name="discount_benchmark", backend="local/csv", root_dir=current_dir
-    )
-    return dataset
+    dataset_path = os.path.join(current_dir, "datasets", "discount_benchmark.csv")
+    # Download dataset from GitHub if it doesn't exist locally
+    if not os.path.exists(dataset_path):
+        os.makedirs(os.path.dirname(dataset_path), exist_ok=True)
+        urllib.request.urlretrieve("https://raw.githubusercontent.com/explodinggradients/ragas/main/examples/ragas_examples/benchmark_llm/datasets/discount_benchmark.csv", dataset_path)
+    return Dataset.load(name="discount_benchmark", backend="local/csv", root_dir=current_dir)
 
 
 def compare_inputs_to_output(
@@ -98,9 +95,9 @@ def compare_inputs_to_output(
     experiment_names = []
     for path in inputs:
         df = pd.read_csv(path)
-        if "experiment_name" not in df.columns:
-            raise ValueError(f"Missing 'experiment_name' column in {path}")
-        exp_name = str(df["experiment_name"].iloc[0])
+        if "model" not in df.columns:
+            raise ValueError(f"Missing 'model' column in {path}")
+        exp_name = str(df["model"].iloc[0])
         experiment_names.append(exp_name)
         dataframes.append(df)
 
@@ -206,11 +203,12 @@ async def run_command(model: str, name: Optional[str]) -> None:
     experiments_dir = os.path.join(current_dir, "experiments")
     os.makedirs(experiments_dir, exist_ok=True)
 
-    benchmark_experiment = create_benchmark_experiment(
-        model_name=model, experiment_name=exp_name
-    )
     print(f"Running model evaluation ({model})...")
-    results = await benchmark_experiment.arun(dataset, name=f"{run_id}-{exp_name}")
+    results = await benchmark_experiment.arun(
+        dataset, 
+        name=f"{run_id}-{exp_name}",
+        model_name=model
+    )
     print(f"✅ {exp_name}: {len(results)} cases evaluated")
     print(f"Results saved to: {os.path.join(experiments_dir, results.name)}.csv")
 

{ragas_examples-0.3.6 → ragas_examples-0.3.7}/ragas_examples/benchmark_llm/prompt.py

@@ -1,12 +1,16 @@
 import os
 
-from openai import OpenAI
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+# Load environment variables
+load_dotenv(".env")
 
 DEFAULT_MODEL = "gpt-4.1-nano-2025-04-14"
 
 
-def get_client() -> OpenAI:
-    """Lazily create an OpenAI client, requiring the API key only when used.
+def get_client() -> AsyncOpenAI:
+    """Lazily create an AsyncOpenAI client, requiring the API key only when used.
 
     This avoids raising errors during module import (e.g., when running --help).
     """
@@ -15,7 +19,7 @@ def get_client() -> OpenAI:
         raise RuntimeError(
             "OPENAI_API_KEY is not set. Please export it before running prompts."
         )
-    return OpenAI(api_key=api_key)
+    return AsyncOpenAI(api_key=api_key)
 
 
 SYSTEM_PROMPT = """
@@ -38,10 +42,10 @@ Respond in JSON format only:
 """
 
 
-def run_prompt(prompt: str, model: str = DEFAULT_MODEL):
+async def run_prompt(prompt: str, model: str = DEFAULT_MODEL):
     """Run the discount calculation prompt with the specified model."""
     client = get_client()
-    response = client.chat.completions.create(
+    response = await client.chat.completions.create(
         model=model,
         response_format={"type": "json_object"},
         messages=[
@@ -54,18 +58,23 @@ def run_prompt(prompt: str, model: str = DEFAULT_MODEL):
 
 
 if __name__ == "__main__":
-    customer_profile = """
-    Customer Profile:
-    - Name: Sarah Johnson
-    - Age: 67
-    - Student: No
-    - Annual Income: $45,000
-    - Premium Member: Yes, for 3 years
-    - Account Age: 3 years
-    """
-    print("=== System Prompt ===")
-    print(SYSTEM_PROMPT)
-    print("\n=== Customer Profile ===")
-    print(customer_profile)
-    print(f"\n=== Running Prompt with default model {DEFAULT_MODEL} ===")
-    print(run_prompt(customer_profile, model=DEFAULT_MODEL))
+    import asyncio
+    
+    async def main():
+        customer_profile = """
+        Customer Profile:
+        - Name: Sarah Johnson
+        - Age: 67
+        - Student: No
+        - Annual Income: $45,000
+        - Premium Member: Yes, for 3 years
+        - Account Age: 3 years
+        """
+        print("=== System Prompt ===")
+        print(SYSTEM_PROMPT)
+        print("\n=== Customer Profile ===")
+        print(customer_profile)
+        print(f"\n=== Running Prompt with default model {DEFAULT_MODEL} ===")
+        print(await run_prompt(customer_profile, model=DEFAULT_MODEL))
+    
+    asyncio.run(main())

{ragas_examples-0.3.6 → ragas_examples-0.3.7}/ragas_examples/improve_rag/evals.py

@@ -75,21 +75,6 @@ def create_ragas_dataset(dataset_path: Path) -> Dataset:
     return dataset
 
 
-# Initialize OpenAI client and models
-def get_openai_client():
-    """Get AsyncOpenAI client with proper error handling."""
-    api_key = os.environ.get("OPENAI_API_KEY")
-    if not api_key:
-        raise ValueError(
-            "OPENAI_API_KEY environment variable is not set. "
-            "Please set your OpenAI API key: export OPENAI_API_KEY='your_key'"
-        )
-    return AsyncOpenAI(api_key=api_key)
-
-def get_llm_client():
-    """Get LLM client for metrics evaluation."""
-    openai_client = get_openai_client()
-    return instructor_llm_factory("openai", model="gpt-5-mini", client=openai_client)
 
 
 def construct_mlflow_trace_url(trace_id: str, mlflow_host: str = "http://127.0.0.1:5000") -> str:
@@ -194,29 +179,34 @@ async def run_experiment(mode: str = "naive", model: str = "gpt-5-mini", name: O
     Returns:
         List of experiment results
     """
-    # Download and prepare dataset
-    dataset_path = download_and_save_dataset()
-    dataset = create_ragas_dataset(dataset_path)
+    # Check for OpenAI API key
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "OPENAI_API_KEY environment variable is not set. "
+            "Please set your OpenAI API key: export OPENAI_API_KEY='your_key'"
+        )
     
-    # Initialize RAG and LLM clients
+    # Prepare dataset and initialize system
     logger.info("Initializing RAG system...")
-    openai_client = get_openai_client()
-    retriever = BM25Retriever()
-    rag = RAG(llm_client=openai_client, retriever=retriever, model=model, mode=mode)
-    llm = get_llm_client()
+    dataset = create_ragas_dataset(download_and_save_dataset())
+    
+    # Initialize RAG system with inline client creation
+    openai_client = AsyncOpenAI(api_key=api_key)
+    rag = RAG(
+        llm_client=openai_client, 
+        retriever=BM25Retriever(), 
+        model=model, 
+        mode=mode
+    )
     logger.info("RAG system initialized!")
     
-    # Generate experiment name if not provided
-    if name is None:
-        mode_label = "agenticrag" if mode == "agentic" else "naiverag"
-        name = f"{datetime.now().strftime('%Y%m%d-%H%M%S')}_{mode_label}"
-    
     # Run evaluation experiment
     experiment_results = await evaluate_rag.arun(
         dataset, 
-        name=name,
+        name=name or f"{datetime.now().strftime('%Y%m%d-%H%M%S')}_{'agenticrag' if mode == 'agentic' else 'naiverag'}",
         rag=rag,
-        llm=llm
+        llm=instructor_llm_factory("openai", model="gpt-5-mini", client=openai_client)
     )
     
     # Print basic results
@@ -230,28 +220,18 @@ async def run_experiment(mode: str = "naive", model: str = "gpt-5-mini", name: O
     return experiment_results
 
 
-async def main(agentic_rag: bool = False):
-    """
-    Main function to run the evaluation pipeline.
-    
-    Args:
-        agentic_rag: If True, use agentic RAG; otherwise use naive RAG
-    """
-    mode = "agentic" if agentic_rag else "naive"
-    return await run_experiment(mode=mode, model="gpt-5-mini")
-
-
 if __name__ == "__main__":
     import sys
     
     # Simple command line argument parsing
     agentic_mode = "--agentic" in sys.argv
+    mode = "agentic" if agentic_mode else "naive"
     
     if agentic_mode:
         logger.info("Running in AGENTIC mode")
     else:
         logger.info("Running in NAIVE mode")
     
-    asyncio.run(main(agentic_rag=agentic_mode))
+    asyncio.run(run_experiment(mode=mode, model="gpt-5-mini"))
 
 

ragas_examples-0.3.7/ragas_examples/judge_alignment/__init__.py

@@ -0,0 +1,29 @@
+"""LLM-as-judge alignment evaluation example.
+
+Functions:
+- load_dataset: Load annotated dataset with human judgments
+- judge_experiment: Run evaluation (Judge → Compare)
+- judge_alignment: Alignment metric comparing judge and human labels
+
+Metrics:
+- accuracy_metric: Baseline judge metric
+- accuracy_metric_v2: Improved judge metric with few-shot examples
+"""
+
+from .evals import (
+    load_dataset,
+    judge_experiment,
+    judge_alignment,
+    accuracy_metric,
+    accuracy_metric_v2,
+)
+
+__all__ = [
+    "load_dataset",
+    "judge_experiment",
+    "judge_alignment",
+    "accuracy_metric",
+    "accuracy_metric_v2",
+]
+
+

ragas_examples-0.3.7/ragas_examples/judge_alignment/evals.py

@@ -0,0 +1,229 @@
+"""
+LLM-as-Judge alignment evaluation example.
+
+Evaluates how well an LLM judge aligns with human judgments by:
+- Using pre-existing responses from the dataset
+- LLM judge evaluates each response
+- Measuring alignment between judge and human labels
+"""
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import pandas as pd
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from ragas import Dataset, experiment
+from ragas.llms import instructor_llm_factory
+from ragas.metrics import DiscreteMetric
+from ragas.metrics.discrete import discrete_metric
+from ragas.metrics.result import MetricResult
+
+load_dotenv()
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(message)s")
+logger = logging.getLogger(__name__)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("openai._base_client").setLevel(logging.WARNING)
+
+
+# Define baseline judge metric with simple prompt
+accuracy_metric = DiscreteMetric(
+    name="accuracy",
+    prompt="Check if the response contains points mentioned from the grading notes and return 'pass' or 'fail'.\n\nResponse: {response}\nGrading Notes: {grading_notes}",
+    allowed_values=["pass", "fail"],
+)
+
+# Define improved judge metric with enhanced evaluation criteria
+accuracy_metric_v2 = DiscreteMetric(
+    name="accuracy",
+    prompt="""Evaluate if the response covers ALL the key concepts from the grading notes. Accept semantic equivalents but carefully check for missing concepts.
+
+ABBREVIATION GUIDE - decode these correctly:
+• Financial: val=valuation, post-$=post-money, rev=revenue, ARR/MRR=Annual/Monthly Recurring Revenue, COGS=Cost of Goods Sold, Opex=Operating Expenses, LTV=Lifetime Value, CAC=Customer Acquisition Cost
+• Business: mkt=market, reg/regs=regulation/regulatory, corp gov=corporate governance, integr=integration, S&M=Sales & Marketing, R&D=Research & Development, acq=acquisition
+• Technical: sys=system, elim=elimination, IP=Intellectual Property, TAM=Total Addressable Market, diff=differentiation
+• Metrics: NPS=Net Promoter Score, SROI=Social Return on Investment, proj=projection, cert=certification
+
+EVALUATION APPROACH:
+
+Step 1 - Parse grading notes into distinct concepts:
+- Separate by commas, semicolons, or line breaks
+- Each item is a concept that must be verified
+- Example: "*Gross Margin* >40%, CAC, LTV:CAC >3:1" = 3 concepts
+
+Step 2 - For each concept, check if it's addressed:
+- Accept semantic equivalents (e.g., "customer acquisition cost" = "CAC")
+- Accept implicit coverage when it's clear (e.g., "revenue forecasting" covers "historical vs forecasted rev")
+- Be flexible on exact numbers (e.g., "around 40%" acceptable for ">40%")
+
+Step 3 - Count missing concepts:
+- Missing 0 concepts = PASS
+- Missing 1+ concepts = FAIL (even one genuinely missing concept should fail)
+- Exception: If a long list (10+ items) has 1 very minor detail missing but all major points covered, use judgment
+
+CRITICAL RULES:
+
+1. Do NOT require exact wording - "market demand" = "mkt demand" = "demand analysis"
+
+2. Markers (* or !) mean important, not mandatory exact phrases:
+   - "*traction evidence*" can be satisfied by discussing metrics, growth, or validation
+   - "!unbiased assumptions" can be satisfied by discussing assumption methodology
+
+3. Numbers should be mentioned but accept approximations:
+   - "$47B to $10B" can be "$47 billion dropped to around $10 billion"
+   - "LTV:CAC >3:1" can be "LTV to CAC ratio of at least 3 to 1" or "3x or higher"
+
+4. FAIL only when concepts are genuinely absent:
+   - If notes mention "liquidation prefs, anti-dilution, board seats" but response only has board seats → FAIL
+   - If notes mention "scalability, tech debt, IP" but response never discusses technical risks → FAIL
+   - If notes mention "GDPR compliance" and response never mentions GDPR or EU regulations → FAIL
+
+5. PASS when ALL concepts present:
+   - All concepts covered, even with different wording → PASS
+   - Concepts addressed implicitly when clearly implied → PASS
+   - Minor phrasing differences → PASS
+   - One or more concepts genuinely absent → FAIL
+
+Response: {response}
+
+Grading Notes: {grading_notes}
+
+Are ALL distinct concepts from the grading notes covered in the response (accepting semantic equivalents and implicit coverage)?""",
+    allowed_values=["pass", "fail"],
+)
+
+
+def load_dataset(csv_path: Optional[Path] = None) -> Dataset:
+    """Load annotated dataset with human judgments.
+    
+    Expected columns: question, grading_notes, response, target (pass/fail)
+    """
+    path = csv_path or (Path(__file__).resolve().parent / "datasets" / "benchmark_df.csv")
+    df = pd.read_csv(path)
+
+    dataset = Dataset(name="llm_judge_alignment", backend="local/csv", root_dir=".")
+    
+    for _, row in df.iterrows():
+        dataset.append({
+            "question": row["question"],
+            "grading_notes": row["grading_notes"],
+            "response": row["response"],
+            "target": str(row["target"]).strip().lower(),
+        })
+    
+    return dataset
+
+
+@discrete_metric(name="judge_alignment", allowed_values=["pass", "fail"])
+def judge_alignment(judge_label: str, human_label: str) -> MetricResult:
+    """Compare judge decision with human label."""
+    judge = judge_label.strip().lower()
+    human = human_label.strip().lower()
+    
+    if judge == human:
+        return MetricResult(value="pass", reason=f"Judge={judge}; Human={human}")
+    
+    return MetricResult(value="fail", reason=f"Judge={judge}; Human={human}")
+
+
+@experiment()
+async def judge_experiment(
+    row: Dict[str, Any],
+    accuracy_metric: DiscreteMetric,
+    llm,
+):
+    """Run complete evaluation: Judge → Compare with human."""
+    # Step 1: Get response (in production, this is where you'd call your LLM app)
+    # For this evaluation, we use pre-existing responses from the dataset
+    app_response = row["response"]
+    
+    # Step 2: Judge evaluates the response
+    judge_score = await accuracy_metric.ascore(
+        question=row["question"],
+        grading_notes=row["grading_notes"],
+        response=app_response,
+        llm=llm,
+    )
+
+    # Step 3: Compare judge decision with human target
+    alignment = judge_alignment.score(
+        judge_label=judge_score.value,
+        human_label=row["target"]
+    )
+
+    return {
+        **row,
+        "judge_label": judge_score.value,
+        "judge_critique": judge_score.reason,
+        "alignment": alignment.value,
+        "alignment_reason": alignment.reason,
+    }
+
+
+async def main():
+    """Example: evaluate judge with baseline prompt."""
+    # Load dataset
+    dataset = load_dataset()
+    logger.info(f"Loaded dataset with {len(dataset)} samples")
+    
+    # Initialize LLM client
+    logger.info("Initializing LLM client with model: gpt-4o-mini")
+    openai_client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+    llm = instructor_llm_factory("openai", model="gpt-4o-mini", client=openai_client)
+    
+    # Run baseline evaluation
+    logger.info("Running baseline evaluation...")
+    results = await judge_experiment.arun(
+        dataset,
+        name="judge_baseline_v1_gpt-4o-mini",
+        accuracy_metric=accuracy_metric,
+        llm=llm,
+    )
+    
+    passed = sum(1 for r in results if r["alignment"] == "pass")
+    total = len(results)
+    logger.info(f"✅ Baseline alignment: {passed}/{total} passed ({passed/total:.1%})")
+    
+    return results
+
+
+async def main_v2():
+    """Evaluate judge with improved v2 prompt."""
+    # Load dataset
+    dataset = load_dataset()
+    logger.info(f"Loaded dataset with {len(dataset)} samples")
+    
+    # Initialize LLM client
+    logger.info("Initializing LLM client with model: gpt-4o-mini")
+    openai_client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+    llm = instructor_llm_factory("openai", model="gpt-4o-mini", client=openai_client)
+    
+    # Run v2 evaluation with improved prompt
+    logger.info("Running v2 evaluation with improved prompt...")
+    results = await judge_experiment.arun(
+        dataset,
+        name="judge_accuracy_v2_gpt-4o-mini",
+        accuracy_metric=accuracy_metric_v2,
+        llm=llm,
+    )
+    
+    passed = sum(1 for r in results if r["alignment"] == "pass")
+    total = len(results)
+    logger.info(f"✅ V2 alignment: {passed}/{total} passed ({passed/total:.1%})")
+    
+    return results
+
+
+if __name__ == "__main__":
+    import asyncio
+    import sys
+    
+    # Run v2 if --v2 flag is passed, otherwise run baseline
+    if len(sys.argv) > 1 and sys.argv[1] == "--v2":
+        asyncio.run(main_v2())
+    else:
+        asyncio.run(main())