ragas-examples 0.3.5__py3-none-any.whl

ragas_examples/benchmark_llm/evals.py

@@ -0,0 +1,274 @@
+import argparse
+import asyncio
+import datetime
+import json
+import os
+import sys
+from typing import List, Optional
+
+import pandas as pd
+
+from ragas import experiment
+from ragas.dataset import Dataset
+from ragas.metrics.discrete import discrete_metric
+from ragas.metrics.result import MetricResult
+
+from .prompt import DEFAULT_MODEL, run_prompt
+
+
+@discrete_metric(name="discount_accuracy", allowed_values=["correct", "incorrect"])
+def discount_accuracy(prediction: str, expected_discount):
+    """Check if the discount prediction is correct."""
+    parsed_json = json.loads(prediction)
+    predicted_discount = parsed_json.get("discount_percentage")
+    expected_discount_int = int(expected_discount)
+
+    if predicted_discount == expected_discount_int:
+        return MetricResult(
+            value="correct",
+            reason=f"Correctly calculated discount={expected_discount_int}%",
+        )
+    else:
+        return MetricResult(
+            value="incorrect",
+            reason=f"Expected discount={expected_discount_int}%; Got discount={predicted_discount}%",
+        )
+
+
+def create_benchmark_experiment(model_name: str, experiment_name: str):
+    """Factory function to create experiment functions for different models."""
+
+    @experiment()
+    async def benchmark_experiment(row):
+        # Get model response (run blocking call in thread to keep async runner responsive)
+        response = await asyncio.to_thread(
+            run_prompt, row["customer_profile"], model=model_name
+        )
+
+        # Parse response (strict JSON mode expected)
+        try:
+            parsed_json = json.loads(response)
+            predicted_discount = parsed_json.get("discount_percentage")
+        except Exception:
+            predicted_discount = None
+
+        # Score the response
+        score = discount_accuracy.score(
+            prediction=response, expected_discount=row["expected_discount"]
+        )
+
+        return {
+            **row,
+            "model": model_name,
+            "experiment_name": experiment_name,
+            "response": response,
+            "predicted_discount": predicted_discount,
+            "score": score.value,
+            "score_reason": score.reason,
+        }
+
+    return benchmark_experiment
+
+
+def load_dataset():
+    """Load the dataset from CSV file."""
+    # Get the directory where this file is located
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+
+    dataset = Dataset.load(
+        name="discount_benchmark", backend="local/csv", root_dir=current_dir
+    )
+    return dataset
+
+
+def compare_inputs_to_output(
+    inputs: List[str], output_path: Optional[str] = None
+) -> str:
+    """Compare multiple experiment CSVs and write a combined CSV.
+
+    - Requires 'id' column in all inputs; uses it as the alignment key
+    - Builds output with id + canonical columns + per-experiment response/score/reason columns
+    - Returns the full output path
+    """
+    if not inputs or len(inputs) < 2:
+        raise ValueError("At least two input CSV files are required for comparison")
+
+    # Load all inputs
+    dataframes = []
+    experiment_names = []
+    for path in inputs:
+        df = pd.read_csv(path)
+        if "experiment_name" not in df.columns:
+            raise ValueError(f"Missing 'experiment_name' column in {path}")
+        exp_name = str(df["experiment_name"].iloc[0])
+        experiment_names.append(exp_name)
+        dataframes.append(df)
+
+    canonical_cols = ["customer_profile", "description", "expected_discount"]
+    base_df = dataframes[0]
+
+    # Require 'id' in all inputs
+    if not all("id" in df.columns for df in dataframes):
+        raise ValueError(
+            "All input CSVs must contain an 'id' column to align rows. Re-run experiments after adding 'id' to your dataset."
+        )
+
+    # Validate duplicates and matching sets of IDs
+    key_sets = []
+    for idx, df in enumerate(dataframes):
+        keys = df["id"].astype(str)
+        if keys.duplicated().any():
+            dupes = keys[keys.duplicated()].head(3).tolist()
+            raise ValueError(
+                f"Input {inputs[idx]} contains duplicate id values. Examples: {dupes}"
+            )
+        key_sets.append(set(keys.tolist()))
+
+    base_keys = key_sets[0]
+    for i, ks in enumerate(key_sets[1:], start=1):
+        if ks != base_keys:
+            missing_in_other = list(base_keys - ks)[:5]
+            missing_in_base = list(ks - base_keys)[:5]
+            raise ValueError(
+                "Inputs do not contain the same set of IDs.\n"
+                f"- Missing in file {i + 1}: {missing_in_other}\n"
+                f"- Extra in file {i + 1}: {missing_in_base}"
+            )
+
+    # Validate canonical columns exist in base
+    missing = [c for c in canonical_cols if c not in base_df.columns]
+    if missing:
+        raise ValueError(f"First CSV missing required columns: {missing}")
+
+    # Build combined on base order using 'id' as alignment key
+    base_ids_str = base_df["id"].astype(str)
+    combined = base_df[["id"] + canonical_cols].copy()
+
+    # Append per-experiment outputs by aligned ID
+    for df, exp_name in zip(dataframes, experiment_names):
+        df = df.copy()
+        df["id"] = df["id"].astype(str)
+        df = df.set_index("id")
+        for col in ["response", "score", "score_reason"]:
+            if col not in df.columns:
+                raise ValueError(
+                    f"Column '{col}' not found in one input. Please provide per-row '{col}'."
+                )
+        combined[f"{exp_name}_response"] = base_ids_str.map(df["response"])
+        combined[f"{exp_name}_score"] = base_ids_str.map(df["score"])
+        combined[f"{exp_name}_score_reason"] = base_ids_str.map(df["score_reason"])
+
+    # Determine output path
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    experiments_dir = os.path.join(current_dir, "experiments")
+    os.makedirs(experiments_dir, exist_ok=True)
+
+    if output_path is None or output_path.strip() == "":
+        run_id = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+        output_path = os.path.join(experiments_dir, f"{run_id}-comparison.csv")
+    else:
+        # If relative path, place under experiments dir
+        if not os.path.isabs(output_path):
+            output_path = os.path.join(experiments_dir, output_path)
+
+    # Sort by id for user-friendly reading
+    if "id" in combined.columns:
+        combined = combined.sort_values(by="id").reset_index(drop=True)
+    combined.to_csv(output_path, index=False)
+
+    # Print per-experiment accuracy summary
+    for df, exp_name in zip(dataframes, experiment_names):
+        try:
+            acc = (df["score"] == "correct").mean()
+            print(f"{exp_name} Accuracy: {acc:.2%}")
+        except Exception:
+            pass
+
+    return output_path
+
+
+async def run_command(model: str, name: Optional[str]) -> None:
+    """Run a single experiment using the provided model and name."""
+    if "OPENAI_API_KEY" not in os.environ:
+        print("❌ Error: OpenAI API key not found!")
+        print("Please set your API key: export OPENAI_API_KEY=your_actual_key")
+        return
+
+    print("Loading dataset...")
+    dataset = load_dataset()
+    print(f"Dataset loaded with {len(dataset)} samples")
+
+    run_id = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    exp_name = name or model
+
+    # Ensure output directory exists (experiment framework saves under experiments/)
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    experiments_dir = os.path.join(current_dir, "experiments")
+    os.makedirs(experiments_dir, exist_ok=True)
+
+    benchmark_experiment = create_benchmark_experiment(
+        model_name=model, experiment_name=exp_name
+    )
+    print(f"Running model evaluation ({model})...")
+    results = await benchmark_experiment.arun(dataset, name=f"{run_id}-{exp_name}")
+    print(f"✅ {exp_name}: {len(results)} cases evaluated")
+    print(f"Results saved to: {os.path.join(experiments_dir, results.name)}.csv")
+
+    # Accuracy summary
+    accuracy = sum(1 for r in results if r["score"] == "correct") / max(1, len(results))
+    print(f"{exp_name} Accuracy: {accuracy:.2%}")
+
+
+def compare_command(inputs: List[str], output: Optional[str]) -> None:
+    output_path = compare_inputs_to_output(inputs, output)
+    print(f"Combined comparison saved to: {output_path}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Benchmark LLM evaluation CLI")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # run subcommand
+    run_parser = subparsers.add_parser("run", help="Run a single experiment")
+    run_parser.add_argument(
+        "--model", type=str, default=DEFAULT_MODEL, help="Model name to evaluate"
+    )
+    run_parser.add_argument(
+        "--name",
+        type=str,
+        default=None,
+        help="Experiment name (defaults to model name)",
+    )
+
+    # compare subcommand
+    cmp_parser = subparsers.add_parser(
+        "compare", help="Combine multiple experiment CSVs"
+    )
+    cmp_parser.add_argument(
+        "--inputs", nargs="+", required=True, help="Input CSV files to compare"
+    )
+    cmp_parser.add_argument(
+        "--output",
+        type=str,
+        default=None,
+        help="Output CSV path (defaults to experiments/<timestamp>-comparison.csv)",
+    )
+
+    return parser
+
+
+if __name__ == "__main__":
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.command == "run":
+        import asyncio
+
+        asyncio.run(run_command(model=args.model, name=args.name))
+        sys.exit(0)
+    elif args.command == "compare":
+        compare_command(inputs=args.inputs, output=args.output)
+        sys.exit(0)
+    else:
+        parser.print_help()
+        sys.exit(2)

ragas_examples/benchmark_llm/prompt.py

@@ -0,0 +1,71 @@
+import os
+
+from openai import OpenAI
+
+DEFAULT_MODEL = "gpt-4.1-nano-2025-04-14"
+
+
+def get_client() -> OpenAI:
+    """Lazily create an OpenAI client, requiring the API key only when used.
+
+    This avoids raising errors during module import (e.g., when running --help).
+    """
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY is not set. Please export it before running prompts."
+        )
+    return OpenAI(api_key=api_key)
+
+
+SYSTEM_PROMPT = """
+You are a discount calculation assistant. I will provide a customer profile and you must calculate their discount percentage and explain your reasoning.
+
+Discount rules:
+- Age 65+ OR student status: 15% discount
+- Annual income < $30,000: 20% discount  
+- Premium member for 2+ years: 10% discount
+- New customer (< 6 months): 5% discount
+
+Rules can stack up to a maximum of 35% discount.
+
+Respond in JSON format only:
+{
+  "discount_percentage": number,
+  "reason": "clear explanation of which rules apply and calculations",
+  "applied_rules": ["list", "of", "applied", "rule", "names"]
+}
+"""
+
+
+def run_prompt(prompt: str, model: str = DEFAULT_MODEL):
+    """Run the discount calculation prompt with the specified model."""
+    client = get_client()
+    response = client.chat.completions.create(
+        model=model,
+        response_format={"type": "json_object"},
+        messages=[
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    response = response.choices[0].message.content.strip()
+    return response
+
+
+if __name__ == "__main__":
+    customer_profile = """
+    Customer Profile:
+    - Name: Sarah Johnson
+    - Age: 67
+    - Student: No
+    - Annual Income: $45,000
+    - Premium Member: Yes, for 3 years
+    - Account Age: 3 years
+    """
+    print("=== System Prompt ===")
+    print(SYSTEM_PROMPT)
+    print("\n=== Customer Profile ===")
+    print(customer_profile)
+    print(f"\n=== Running Prompt with default model {DEFAULT_MODEL} ===")
+    print(run_prompt(customer_profile, model=DEFAULT_MODEL))

ragas_examples/prompt_evals/evals.py

@@ -0,0 +1,75 @@
+from ragas import Dataset, experiment
+from ragas.metrics.discrete import discrete_metric
+from ragas.metrics.result import MetricResult
+
+from .prompt import run_prompt
+
+
+@discrete_metric(name="accuracy", allowed_values=["pass", "fail"])
+def my_metric(prediction: str, actual: str):
+    """Calculate accuracy of the prediction."""
+    return (
+        MetricResult(value="pass", reason="")
+        if prediction == actual
+        else MetricResult(value="fail", reason="")
+    )
+
+
+@experiment()
+async def run_experiment(row):
+    response = run_prompt(row["text"])
+    score = my_metric.score(prediction=response, actual=row["label"])
+
+    experiment_view = {
+        **row,
+        "response": response,
+        "score": score.value,
+    }
+    return experiment_view
+
+
+def load_dataset():
+    # Create a dataset
+    dataset = Dataset(
+        name="test_dataset",
+        backend="local/csv",
+        root_dir=".",
+    )
+    dataset_dict = [
+        {"text": "I loved the movie! It was fantastic.", "label": "positive"},
+        {"text": "The movie was terrible and boring.", "label": "negative"},
+        {"text": "It was an average film, nothing special.", "label": "positive"},
+        {"text": "Absolutely amazing! Best movie of the year.", "label": "positive"},
+        {"text": "I did not like it at all, very disappointing.", "label": "negative"},
+        {"text": "It was okay, not the best but not the worst.", "label": "positive"},
+        {
+            "text": "I have mixed feelings about it, some parts were good, others not so much.",
+            "label": "positive",
+        },
+        {"text": "What a masterpiece! I would watch it again.", "label": "positive"},
+        {
+            "text": "I would not recommend it to anyone, it was that bad.",
+            "label": "negative",
+        },
+    ]
+
+    for sample in dataset_dict:
+        row = {"text": sample["text"], "label": sample["label"]}
+        dataset.append(row)
+
+    # make sure to save it
+    dataset.save()
+    return dataset
+
+
+async def main():
+    dataset = load_dataset()
+    experiment_results = await run_experiment.arun(dataset)
+    print("Experiment completed successfully!")
+    print("Experiment results:", experiment_results)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

ragas_examples/prompt_evals/prompt.py

@@ -0,0 +1,32 @@
+import os
+
+from openai import OpenAI
+
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+
+SYSTEM_PROMPT = """
+You are a helpful assistant. I will provide a movie review and you will classify it as either positive or negative.
+Please respond with "positive" or "negative" only.
+"""
+
+
+def run_prompt(prompt: str):
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    response = (
+        response.choices[0].message.content.strip()
+        if response.choices[0].message.content
+        else ""
+    )
+    return response
+
+
+if __name__ == "__main__":
+    prompt = "The movie was fantastic and I loved every moment of it!"
+    print(run_prompt(prompt))

ragas_examples/rag_eval/evals.py

@@ -0,0 +1,84 @@
+import os
+
+from openai import OpenAI
+
+from ragas import Dataset, experiment
+from ragas.llms import instructor_llm_factory
+from ragas.metrics import DiscreteMetric
+
+from .rag import default_rag_client
+
+openai_client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+rag_client = default_rag_client(llm_client=openai_client)
+llm = instructor_llm_factory("openai", model="gpt-4o", client=openai_client)
+
+
+def load_dataset():
+    dataset = Dataset(
+        name="test_dataset",
+        backend="local/csv",
+        root_dir=".",
+    )
+
+    data_samples = [
+        {
+            "question": "What is ragas 0.3",
+            "grading_notes": "- experimentation as the central pillar - provides abstraction for datasets, experiments and metrics - supports evals for RAG, LLM workflows and Agents",
+        },
+        {
+            "question": "how are experiment results stored in ragas 0.3?",
+            "grading_notes": "- configured using different backends like local, gdrive, etc - stored under experiments/ folder in the backend storage",
+        },
+        {
+            "question": "What metrics are supported in ragas 0.3?",
+            "grading_notes": "- provides abstraction for discrete, numerical and ranking metrics",
+        },
+    ]
+
+    for sample in data_samples:
+        row = {"question": sample["question"], "grading_notes": sample["grading_notes"]}
+        dataset.append(row)
+
+    # make sure to save it
+    dataset.save()
+    return dataset
+
+
+my_metric = DiscreteMetric(
+    name="correctness",
+    prompt="Check if the response contains points mentioned from the grading notes and return 'pass' or 'fail'.\nResponse: {response} Grading Notes: {grading_notes}",
+    allowed_values=["pass", "fail"],
+)
+
+
+@experiment()
+async def run_experiment(row):
+    response = rag_client.query(row["question"])
+
+    score = my_metric.score(
+        llm=llm,
+        response=response.get("answer", " "),
+        grading_notes=row["grading_notes"],
+    )
+
+    experiment_view = {
+        **row,
+        "response": response.get("answer", ""),
+        "score": score.value,
+        "log_file": response.get("logs", " "),
+    }
+    return experiment_view
+
+
+async def main():
+    dataset = load_dataset()
+    print("dataset loaded successfully", dataset)
+    experiment_results = await run_experiment.arun(dataset)
+    print("Experiment completed successfully!")
+    print("Experiment results:", experiment_results)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())