trulens-benchmark 1.0.1a1__tar.gz

trulens_benchmark-1.0.1a1/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: trulens-benchmark
+Version: 1.0.1a1
+Summary: Library to systematically track and evaluate LLM based applications.
+Home-page: https://trulens.org/
+License: MIT
+Author: Snowflake Inc.
+Author-email: ml-observability-wg-dl@snowflake.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: trulens-core (>=1.0.0,<2.0.0)
+Project-URL: Documentation, https://trulens.org/trulens/getting_started/
+Project-URL: Repository, https://github.com/truera/trulens
+Description-Content-Type: text/markdown
+
+# trulens-benchmark
+

trulens_benchmark-1.0.1a1/README.md

@@ -0,0 +1 @@
+# trulens-benchmark

trulens_benchmark-1.0.1a1/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = [
+  "poetry-core",
+]
+
+[tool.poetry]
+name = "trulens-benchmark"
+version = "1.0.1a1"
+description = "Library to systematically track and evaluate LLM based applications."
+authors = [
+  "Snowflake Inc. <ml-observability-wg-dl@snowflake.com>",
+]
+license = "MIT"
+readme = "README.md"
+packages = [
+  { include = "trulens" },
+]
+homepage = "https://trulens.org/"
+documentation = "https://trulens.org/trulens/getting_started/"
+repository = "https://github.com/truera/trulens"
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+trulens-core = { version = "^1.0.0", allow-prereleases = true }
+
+[tool.poetry.group.dev.dependencies]
+trulens-core = { path = "../core", develop = true }

trulens_benchmark-1.0.1a1/trulens/benchmark/__init__.py

@@ -0,0 +1,15 @@
+"""
+!!! note "Additional Dependency Required"
+
+    To use this module, you must have the `trulens-benchmark` package installed.
+
+    ```bash
+    pip install trulens-benchmark
+    ```
+"""
+
+from importlib.metadata import version
+
+from trulens.core.utils.imports import safe_importlib_package_name
+
+__version__ = version(safe_importlib_package_name(__package__ or __name__))

trulens_benchmark-1.0.1a1/trulens/benchmark/answer_relevance_benchmark_small.ipynb

@@ -0,0 +1,247 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 📓 Answer Relevance Feedback Evaluation\n",
+    "In many ways, feedbacks can be thought of as LLM apps themselves. Given text,\n",
+    "they return some result. Thinking in this way, we can use _TruLens_ to evaluate\n",
+    "and track our feedback quality. We can even do this for different models (e.g.\n",
+    "gpt-3.5 and gpt-4) or prompting schemes (such as chain-of-thought reasoning).\n",
+    "\n",
+    "This notebook follows an evaluation of a set of test cases. You are encouraged\n",
+    "to run this on your own and even expand the test cases to evaluate performance\n",
+    "on test cases applicable to your scenario or domain."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import relevance feedback function\n",
+    "from test_cases import answer_relevance_golden_set\n",
+    "from trulens.core import Feedback\n",
+    "from trulens.core import Select\n",
+    "from trulens.core import Tru\n",
+    "from trulens.core import TruBasicApp\n",
+    "from trulens.feedback import GroundTruthAgreement\n",
+    "from trulens.providers.litellm import LiteLLM\n",
+    "from trulens.providers.openai import OpenAI\n",
+    "\n",
+    "Tru().reset_database()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n",
+    "os.environ[\"COHERE_API_KEY\"] = \"...\"\n",
+    "os.environ[\"HUGGINGFACE_API_KEY\"] = \"...\"\n",
+    "os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n",
+    "os.environ[\"TOGETHERAI_API_KEY\"] = \"...\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# GPT 3.5\n",
+    "turbo = OpenAI(model_engine=\"gpt-3.5-turbo\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_turbo(input, output):\n",
+    "    return turbo.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# GPT 4\n",
+    "gpt4 = OpenAI(model_engine=\"gpt-4\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_gpt4(input, output):\n",
+    "    return gpt4.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# Cohere\n",
+    "command_nightly = LiteLLM(model_engine=\"cohere/command-nightly\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_command_nightly(input, output):\n",
+    "    return command_nightly.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# Anthropic\n",
+    "claude_1 = LiteLLM(model_engine=\"claude-instant-1\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_claude1(input, output):\n",
+    "    return claude_1.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "claude_2 = LiteLLM(model_engine=\"claude-2\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_claude2(input, output):\n",
+    "    return claude_2.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# Meta\n",
+    "llama_2_13b = LiteLLM(\n",
+    "    model_engine=\"together_ai/togethercomputer/Llama-2-7B-32K-Instruct\"\n",
+    ")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_llama2(input, output):\n",
+    "    return llama_2_13b.relevance(input, output)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here we'll set up our golden set as a set of prompts, responses and expected\n",
+    "scores stored in `test_cases.py`. Then, our numeric_difference method will look\n",
+    "up the expected score for each prompt/response pair by **exact match**. After\n",
+    "looking up the expected score, we will then take the L1 difference between the\n",
+    "actual score and expected score."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a Feedback object using the numeric_difference method of the\n",
+    "# ground_truth object\n",
+    "ground_truth = GroundTruthAgreement(\n",
+    "    answer_relevance_golden_set, provider=OpenAI()\n",
+    ")\n",
+    "\n",
+    "# Call the numeric_difference method with app and record and aggregate to get\n",
+    "# the mean absolute error\n",
+    "f_mae = (\n",
+    "    Feedback(ground_truth.mae, name=\"Mean Absolute Error\")\n",
+    "    .on(Select.Record.calls[0].args.args[0])\n",
+    "    .on(Select.Record.calls[0].args.args[1])\n",
+    "    .on_output()\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tru_wrapped_relevance_turbo = TruBasicApp(\n",
+    "    wrapped_relevance_turbo,\n",
+    "    app_id=\"answer relevance gpt-3.5-turbo\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_gpt4 = TruBasicApp(\n",
+    "    wrapped_relevance_gpt4, app_id=\"answer relevance gpt-4\", feedbacks=[f_mae]\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_commandnightly = TruBasicApp(\n",
+    "    wrapped_relevance_command_nightly,\n",
+    "    app_id=\"answer relevance Command-Nightly\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_claude1 = TruBasicApp(\n",
+    "    wrapped_relevance_claude1,\n",
+    "    app_id=\"answer relevance Claude 1\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_claude2 = TruBasicApp(\n",
+    "    wrapped_relevance_claude2,\n",
+    "    app_id=\"answer relevance Claude 2\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_llama2 = TruBasicApp(\n",
+    "    wrapped_relevance_llama2,\n",
+    "    app_id=\"answer relevance Llama-2-13b\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(len(answer_relevance_golden_set)):\n",
+    "    prompt = answer_relevance_golden_set[i][\"query\"]\n",
+    "    response = answer_relevance_golden_set[i][\"response\"]\n",
+    "\n",
+    "    with tru_wrapped_relevance_turbo as recording:\n",
+    "        tru_wrapped_relevance_turbo.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_gpt4 as recording:\n",
+    "        tru_wrapped_relevance_gpt4.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_commandnightly as recording:\n",
+    "        tru_wrapped_relevance_commandnightly.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_claude1 as recording:\n",
+    "        tru_wrapped_relevance_claude1.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_claude2 as recording:\n",
+    "        tru_wrapped_relevance_claude2.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_llama2 as recording:\n",
+    "        tru_wrapped_relevance_llama2.app(prompt, response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Tru().get_leaderboard(app_ids=[]).sort_values(by=\"Mean Absolute Error\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.11.4 ('agents')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "7d153714b979d5e6d08dd8ec90712dd93bff2c9b6c1f0c118169738af3430cd4"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

trulens_benchmark-1.0.1a1/trulens/benchmark/benchmark_frameworks/eval_as_recommendation.py

@@ -0,0 +1,124 @@
+import logging
+import time
+
+import numpy as np
+from sklearn.metrics import ndcg_score
+
+log = logging.getLogger(__name__)
+"""score passages with feedback function, retrying if feedback function fails.
+    Args: df: dataframe with columns 'query_id', 'query', 'passage', 'is_selected'
+            feedback_func: function that takes query and passage as input and returns a score
+            backoff_time: time to wait between retries
+            n: number of samples to estimate conditional probabilities of feedback_func's scores
+"""
+
+
+def score_passages(
+    df,
+    feedback_func_name,
+    feedback_func,
+    backoff_time=0.5,
+    n=5,
+    temperature=0.0,
+):
+    grouped = df.groupby("query_id")
+    scores = []
+    true_relevance = []
+
+    for name, group in grouped:
+        query_scores = []
+        query_relevance = []
+        for _, row in group.iterrows():
+            sampled_score = None
+            if feedback_func_name == "TruEra" or n == 1:
+                sampled_score = feedback_func(
+                    row["query"], row["passage"], temperature
+                )  # hard-coded for now, we don't need to sample for TruEra BERT-based model
+                time.sleep(backoff_time)
+            else:
+                sampled_scores = []
+                for _ in range(n):
+                    sampled_scores.append(
+                        feedback_func(row["query"], row["passage"], temperature)
+                    )
+                    time.sleep(backoff_time)
+                sampled_score = sum(sampled_scores) / len(sampled_scores)
+            query_scores.append(sampled_score)
+            query_relevance.append(row["is_selected"])
+            # print(f"Feedback avg score for query {name} is {sampled_score}, is_selected is {row['is_selected']}")
+
+        print(
+            f"Feedback function {name} scored {len(query_scores)} out of {len(group)} passages."
+        )
+        scores.append(query_scores)
+        true_relevance.append(query_relevance)
+
+    return scores, true_relevance
+
+
+def compute_ndcg(scores, true_relevance):
+    ndcg_values = [
+        ndcg_score([true], [pred]) for true, pred in zip(true_relevance, scores)
+    ]
+    return np.mean(ndcg_values)
+
+
+def compute_ece(scores, true_relevance, n_bins=10):
+    ece = 0
+    for bin in np.linspace(0, 1, n_bins):
+        bin_scores = []
+        bin_truth = []
+        for score_list, truth_list in zip(scores, true_relevance):
+            for score, truth in zip(score_list, truth_list):
+                if bin <= score < bin + 1 / n_bins:
+                    bin_scores.append(score)
+                    bin_truth.append(truth)
+
+        if bin_scores:
+            bin_avg_confidence = np.mean(bin_scores)
+            bin_accuracy = np.mean(bin_truth)
+            ece += (
+                np.abs(bin_avg_confidence - bin_accuracy)
+                * len(bin_scores)
+                / sum(map(len, scores))
+            )
+
+    return ece
+
+
+def precision_at_k(scores, true_relevance, k):
+    sorted_scores = sorted(scores, reverse=True)
+    kth_score = sorted_scores[min(k - 1, len(scores) - 1)]
+
+    # Indices of items with scores >= kth highest score
+    top_k_indices = [i for i, score in enumerate(scores) if score >= kth_score]
+
+    # Calculate precision
+    true_positives = sum(np.take(true_relevance, top_k_indices))
+    return true_positives / len(top_k_indices) if top_k_indices else 0
+
+
+def recall_at_k(scores, true_relevance, k):
+    """
+    Calculate the recall at K.
+
+    Parameters:
+    true_relevance (list of int): List of binary values indicating relevance (1 for relevant, 0 for not).
+    scores (list of float): List of scores assigned by the model.
+    k (int): Number of top items to consider for calculating recall.
+
+    Returns:
+    float: Recall at K.
+    """
+    sorted_scores = sorted(scores, reverse=True)
+    kth_score = sorted_scores[min(k - 1, len(scores) - 1)]
+
+    # Indices of items with scores >= kth highest score
+    top_k_indices = [i for i, score in enumerate(scores) if score >= kth_score]
+
+    # Calculate recall
+    relevant_indices = np.where(true_relevance)[0]
+    hits = sum(idx in top_k_indices for idx in relevant_indices)
+    total_relevant = sum(true_relevance)
+
+    return hits / total_relevant if total_relevant > 0 else 0