trulens-benchmark 1.0.1__tar.gz

trulens_benchmark-1.0.1/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: trulens-benchmark
+Version: 1.0.1
+Summary: Library to systematically track and evaluate LLM based applications.
+Home-page: https://trulens.org/
+License: MIT
+Author: Snowflake Inc.
+Author-email: ml-observability-wg-dl@snowflake.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: trulens-core (>=1.0.0,<2.0.0)
+Project-URL: Documentation, https://trulens.org/trulens/getting_started/
+Project-URL: Repository, https://github.com/truera/trulens
+Description-Content-Type: text/markdown
+
+# trulens-benchmark
+

trulens_benchmark-1.0.1/README.md

@@ -0,0 +1 @@
+# trulens-benchmark

trulens_benchmark-1.0.1/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = [
+  "poetry-core",
+]
+
+[tool.poetry]
+name = "trulens-benchmark"
+version = "1.0.1"
+description = "Library to systematically track and evaluate LLM based applications."
+authors = [
+  "Snowflake Inc. <ml-observability-wg-dl@snowflake.com>",
+]
+license = "MIT"
+readme = "README.md"
+packages = [
+  { include = "trulens" },
+]
+homepage = "https://trulens.org/"
+documentation = "https://trulens.org/trulens/getting_started/"
+repository = "https://github.com/truera/trulens"
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+trulens-core = { version = "^1.0.0" }
+
+[tool.poetry.group.dev.dependencies]
+trulens-core = { path = "../core", develop = true }

trulens_benchmark-1.0.1/trulens/benchmark/__init__.py

@@ -0,0 +1,15 @@
+"""
+!!! note "Additional Dependency Required"
+
+    To use this module, you must have the `trulens-benchmark` package installed.
+
+    ```bash
+    pip install trulens-benchmark
+    ```
+"""
+
+from importlib.metadata import version
+
+from trulens.core.utils.imports import safe_importlib_package_name
+
+__version__ = version(safe_importlib_package_name(__package__ or __name__))

trulens_benchmark-1.0.1/trulens/benchmark/answer_relevance_benchmark_small.ipynb

@@ -0,0 +1,255 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 📓 Answer Relevance Feedback Evaluation\n",
+    "In many ways, feedbacks can be thought of as LLM apps themselves. Given text,\n",
+    "they return some result. Thinking in this way, we can use _TruLens_ to evaluate\n",
+    "and track our feedback quality. We can even do this for different models (e.g.\n",
+    "gpt-3.5 and gpt-4) or prompting schemes (such as chain-of-thought reasoning).\n",
+    "\n",
+    "This notebook follows an evaluation of a set of test cases. You are encouraged\n",
+    "to run this on your own and even expand the test cases to evaluate performance\n",
+    "on test cases applicable to your scenario or domain."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import relevance feedback function\n",
+    "from test_cases import answer_relevance_golden_set\n",
+    "from trulens.apps.basic import TruBasicApp\n",
+    "from trulens.core import Feedback\n",
+    "from trulens.core import Select\n",
+    "from trulens.core import TruSession\n",
+    "from trulens.feedback import GroundTruthAgreement\n",
+    "from trulens.providers.litellm import LiteLLM\n",
+    "from trulens.providers.openai import OpenAI\n",
+    "\n",
+    "TruSession().reset_database()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n",
+    "os.environ[\"COHERE_API_KEY\"] = \"...\"\n",
+    "os.environ[\"HUGGINGFACE_API_KEY\"] = \"...\"\n",
+    "os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n",
+    "os.environ[\"TOGETHERAI_API_KEY\"] = \"...\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# GPT 3.5\n",
+    "turbo = OpenAI(model_engine=\"gpt-3.5-turbo\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_turbo(input, output):\n",
+    "    return turbo.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# GPT 4\n",
+    "gpt4 = OpenAI(model_engine=\"gpt-4\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_gpt4(input, output):\n",
+    "    return gpt4.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# Cohere\n",
+    "command_nightly = LiteLLM(model_engine=\"cohere/command-nightly\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_command_nightly(input, output):\n",
+    "    return command_nightly.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# Anthropic\n",
+    "claude_1 = LiteLLM(model_engine=\"claude-instant-1\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_claude1(input, output):\n",
+    "    return claude_1.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "claude_2 = LiteLLM(model_engine=\"claude-2\")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_claude2(input, output):\n",
+    "    return claude_2.relevance(input, output)\n",
+    "\n",
+    "\n",
+    "# Meta\n",
+    "llama_2_13b = LiteLLM(\n",
+    "    model_engine=\"together_ai/togethercomputer/Llama-2-7B-32K-Instruct\"\n",
+    ")\n",
+    "\n",
+    "\n",
+    "def wrapped_relevance_llama2(input, output):\n",
+    "    return llama_2_13b.relevance(input, output)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Here we'll set up our golden set as a set of prompts, responses and expected\n",
+    "scores stored in `test_cases.py`. Then, our numeric_difference method will look\n",
+    "up the expected score for each prompt/response pair by **exact match**. After\n",
+    "looking up the expected score, we will then take the L1 difference between the\n",
+    "actual score and expected score."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a Feedback object using the numeric_difference method of the\n",
+    "# ground_truth object\n",
+    "ground_truth = GroundTruthAgreement(\n",
+    "    answer_relevance_golden_set, provider=OpenAI()\n",
+    ")\n",
+    "\n",
+    "# Call the numeric_difference method with app and record and aggregate to get\n",
+    "# the mean absolute error\n",
+    "f_mae = (\n",
+    "    Feedback(ground_truth.mae, name=\"Mean Absolute Error\")\n",
+    "    .on(Select.Record.calls[0].args.args[0])\n",
+    "    .on(Select.Record.calls[0].args.args[1])\n",
+    "    .on_output()\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tru_wrapped_relevance_turbo = TruBasicApp(\n",
+    "    wrapped_relevance_turbo,\n",
+    "    app_name=\"answer relevance\",\n",
+    "    app_version=\"gpt-3.5-turbo\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_gpt4 = TruBasicApp(\n",
+    "    wrapped_relevance_gpt4,\n",
+    "    app_name=\"answer relevance\",\n",
+    "    app_version=\"gpt-4\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_commandnightly = TruBasicApp(\n",
+    "    wrapped_relevance_command_nightly,\n",
+    "    app_name=\"answer relevance\",\n",
+    "    app_version=\"Command-Nightly\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_claude1 = TruBasicApp(\n",
+    "    wrapped_relevance_claude1,\n",
+    "    app_name=\"answer relevance\",\n",
+    "    app_version=\"Claude 1\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_claude2 = TruBasicApp(\n",
+    "    wrapped_relevance_claude2,\n",
+    "    app_name=\"answer relevance\",\n",
+    "    app_version=\"Claude 2\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")\n",
+    "\n",
+    "tru_wrapped_relevance_llama2 = TruBasicApp(\n",
+    "    wrapped_relevance_llama2,\n",
+    "    app_name=\"answer relevance\",\n",
+    "    app_version=\"Llama-2-13b\",\n",
+    "    feedbacks=[f_mae],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(len(answer_relevance_golden_set)):\n",
+    "    prompt = answer_relevance_golden_set[i][\"query\"]\n",
+    "    response = answer_relevance_golden_set[i][\"response\"]\n",
+    "\n",
+    "    with tru_wrapped_relevance_turbo as recording:\n",
+    "        tru_wrapped_relevance_turbo.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_gpt4 as recording:\n",
+    "        tru_wrapped_relevance_gpt4.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_commandnightly as recording:\n",
+    "        tru_wrapped_relevance_commandnightly.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_claude1 as recording:\n",
+    "        tru_wrapped_relevance_claude1.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_claude2 as recording:\n",
+    "        tru_wrapped_relevance_claude2.app(prompt, response)\n",
+    "\n",
+    "    with tru_wrapped_relevance_llama2 as recording:\n",
+    "        tru_wrapped_relevance_llama2.app(prompt, response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TruSession().get_leaderboard().sort_values(by=\"Mean Absolute Error\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.11.4 ('agents')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "7d153714b979d5e6d08dd8ec90712dd93bff2c9b6c1f0c118169738af3430cd4"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

trulens_benchmark-1.0.1/trulens/benchmark/benchmark_frameworks/dataset/beir_loader.py

@@ -0,0 +1,319 @@
+import csv
+import json
+import logging
+import os
+from typing import Any, Dict, Generator, List, Optional, Tuple
+import zipfile
+
+import pandas as pd
+import requests
+from tqdm.autonotebook import tqdm
+from trulens.core import TruSession
+
+logger = logging.getLogger(__name__)
+
+
+def download_and_unzip(url: str, out_dir: str, chunk_size: int = 1024) -> str:
+    def download_url(url: str, save_path: str, chunk_size: int = 1024):
+        """Download url with progress bar using tqdm
+        https://stackoverflow.com/questions/15644964/python-progress-bar-and-downloads
+
+        Args:
+            url (str): downloadable url
+            save_path (str): local path to save the downloaded file
+            chunk_size (int, optional): chunking of files. Defaults to 1024.
+        """
+        r = requests.get(url, stream=True)
+        total = int(r.headers.get("Content-Length", 0))
+        with open(save_path, "wb") as fd, tqdm(
+            desc=save_path,
+            total=total,
+            unit="iB",
+            unit_scale=True,
+            unit_divisor=chunk_size,
+        ) as bar:
+            for data in r.iter_content(chunk_size=chunk_size):
+                size = fd.write(data)
+                bar.update(size)
+
+    def unzip(zip_file: str, out_dir: str):
+        zip_ = zipfile.ZipFile(zip_file, "r")
+        zip_.extractall(path=out_dir)
+        zip_.close()
+
+    os.makedirs(out_dir, exist_ok=True)
+    dataset = url.split("/")[-1]
+    zip_file = os.path.join(out_dir, dataset)
+
+    if not os.path.isfile(zip_file):
+        logger.info("Downloading {} ...".format(dataset))
+        download_url(url, zip_file, chunk_size)
+
+    if not os.path.isdir(zip_file.replace(".zip", "")):
+        logger.info("Unzipping {} ...".format(dataset))
+        unzip(zip_file, out_dir)
+
+    return os.path.join(out_dir, dataset.replace(".zip", ""))
+
+
+class TruBEIRDataLoader:
+    def __init__(
+        self,
+        data_folder: str,
+        dataset_name: str,
+        corpus_file: str = "corpus.jsonl",
+        query_file: str = "queries.jsonl",
+        qrels_folder: str = "qrels",
+        qrels_file: str = "",
+    ):
+        """
+        A utility class to load BEIR datasets into TruLens. Similar to https://github.com/beir-cellar/beir but with slightly
+        more efficient implementation of processing and loading of the datasets using generators.
+
+        Args:
+            data_folder (str): local / remote path to store the downloaded dataset.
+            dataset_name (str): Name of the dataset to be loaded. Must be one of https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/
+            corpus_file (str, optional): file name of the corpus. Defaults to "corpus.jsonl".
+            query_file (str, optional): file name of all the queries. Defaults to "queries.jsonl".
+            qrels_folder (str, optional): folder name of qrels (relevance annotation). Defaults to "qrels".
+            qrels_file (str, optional): file name of qrels (relevance annotation). Defaults to "".
+        """
+        self.qrels = {}
+
+        self.dataset_name = dataset_name
+        self.data_folder = data_folder
+
+        self.corpus_file = (
+            os.path.join(self.data_folder, dataset_name, corpus_file)
+            if self.data_folder
+            else corpus_file
+        )
+        self.query_file = (
+            os.path.join(self.data_folder, dataset_name, query_file)
+            if self.data_folder
+            else query_file
+        )
+        self.qrels_folder = os.path.join(
+            self.data_folder, dataset_name, qrels_folder
+        )
+
+        self.qrels_file = qrels_file
+
+    @staticmethod
+    def check(fIn: str, ext: str):
+        if not os.path.exists(fIn):
+            raise ValueError(
+                f"File {fIn} not present! Please provide accurate file."
+            )
+
+        if not fIn.endswith(ext):
+            raise ValueError(f"File {fIn} must be present with extension {ext}")
+
+    def _load_qrels(self):
+        reader = csv.reader(
+            open(self.qrels_file, encoding="utf-8"),
+            delimiter="\t",
+            quoting=csv.QUOTE_MINIMAL,
+        )
+        next(reader)
+
+        for id, row in enumerate(reader):
+            query_id, corpus_id, score = row[0], row[1], int(row[2])
+
+            if query_id not in self.qrels:
+                self.qrels[query_id] = {corpus_id: score}
+            else:
+                self.qrels[query_id][corpus_id] = score
+
+    def _load_generators(
+        self, split="test"
+    ) -> Tuple[
+        Generator[Dict[str, Dict[str, str]], None, None],
+        Generator[Dict[str, str], None, None],
+    ]:
+        """
+        Load corpus, queries, and qrels as generators.
+
+        Args:
+            split (str, optional): Dataset split to load. Defaults to "test".
+
+        Returns:
+            Tuple of generators for corpus, queries, and qrels.
+        """
+        self.qrels_file = os.path.join(self.qrels_folder, split + ".tsv")
+        self.check(fIn=self.corpus_file, ext="jsonl")
+        self.check(fIn=self.query_file, ext="jsonl")
+        self.check(fIn=self.qrels_file, ext="tsv")
+
+        corpus_gen = self._corpus_generator()
+
+        self._load_qrels()
+        queries_gen = self._queries_generator()
+
+        return corpus_gen, queries_gen
+
+    def _corpus_generator(
+        self,
+    ) -> Generator[Dict[str, Dict[str, str]], None, None]:
+        """
+        Generator to load corpus data incrementally.
+        """
+        self.check(fIn=self.corpus_file, ext="jsonl")
+
+        with open(self.corpus_file, encoding="utf8") as fIn:
+            for line in fIn:
+                line = json.loads(line)
+                yield {
+                    line.get("_id"): {
+                        "text": line.get("text"),
+                        "title": line.get("title"),
+                    }
+                }
+
+    def _queries_generator(self) -> Generator[Dict[str, str], None, None]:
+        """
+        Generator to load queries incrementally.
+        """
+        self.check(fIn=self.query_file, ext="jsonl")
+
+        with open(self.query_file, encoding="utf8") as fIn:
+            for line in fIn:
+                line = json.loads(line)
+                if line.get("_id") in self.qrels:
+                    yield {line.get("_id"): line.get("text")}
+
+    def _process_dataset(
+        self, split="test", chunk_size=None
+    ) -> Generator[List[Dict[str, Any]], None, None]:
+        """
+        Common method to process the BEIR dataset into entries.
+        This method handles downloading, loading, and processing the dataset.
+
+        Args:
+            split (str, optional): Dataset split to load. Defaults to "test".
+            chunk_size (int, optional): Number of records to process in each chunk. Defaults to None.
+
+        Yields:
+            List[Dict[str, Any]]: List of dataset entries (a chunk if chunk_size is specified).
+        """
+
+        url = f"https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{self.dataset_name}.zip"
+
+        logger.info(f"Downloading {self.dataset_name} dataset from {url}")
+        download_and_unzip(url, self.data_folder)
+
+        corpus_gen, queries_gen = self._load_generators(split=split)
+
+        dataset_entries = []
+
+        corpus_dict = {}
+        for corpus in list(
+            corpus_gen
+        ):  # TODO: (Daniel) this is still going to be a memory hog for large datasets, but if we don't
+            # load the entire corpus into memory, we'll have to re-read the entire corpus using generator for each query
+            corpus_dict.update(corpus)
+
+        # Iterate over queries generator and process entries
+        for i, query in enumerate(queries_gen, 1):
+            for query_id, query_text in query.items():
+                doc_to_rel = self.qrels.get(query_id, {})
+
+                expected_chunks = []
+
+                for corpus_id in doc_to_rel.keys():
+                    if corpus_id in corpus_dict:
+                        corpus_entry = corpus_dict[corpus_id]
+                        expected_chunks.append({
+                            "text": corpus_entry["text"],
+                            "title": corpus_entry.get("title"),
+                            "expected_score": doc_to_rel.get(
+                                corpus_id, 0
+                            ),  # Default relevance score to 0 if not found
+                        })
+
+                dataset_entries.append({
+                    "query_id": query_id,
+                    "query": query_text,
+                    "expected_response": None,  # expected response can be empty for IR datasets
+                    "expected_chunks": expected_chunks,
+                    "meta": {"source": self.dataset_name},
+                })
+
+                # Yield the chunk if chunk_size is specified
+                if chunk_size and i % chunk_size == 0:
+                    yield dataset_entries
+                    dataset_entries = []
+
+        if dataset_entries:
+            yield dataset_entries
+
+    def load_dataset_to_df(
+        self,
+        split="test",
+        download=True,
+    ) -> pd.DataFrame:
+        """
+        load BEIR dataset into dataframe with pre-processed fields to match expected TruLens schemas.
+        Note this method loads the entire dataset into memory at once.
+        Args:
+            split (str, optional): Defaults to "test".
+            download (bool, optional): If False, remove the downloaded dataset file after processing. Defaults to True.
+        Returns:
+            pd.DataFrame: DataFrame with the BEIR dataset
+        """
+        dataset_entries = []
+        for chunk in self._process_dataset(split=split):
+            dataset_entries.extend(chunk)
+
+        if not download:
+            logger.info(f"Cleaning up downloaded {self.dataset_name} dataset")
+            os.system(
+                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}"
+            )
+            os.system(
+                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}.zip"
+            )
+
+        return pd.DataFrame(dataset_entries)
+
+    def persist_dataset(
+        self,
+        session: TruSession,
+        dataset_name: str,
+        dataset_metadata: Optional[Dict[str, Any]] = None,
+        split="test",
+        download=True,
+        chunk_size=1000,
+    ):
+        """
+        Persist BEIR dataset into DB with pre-processed fields to match expected TruLens schemas.
+        Note this method handle chunking of the dataset to avoid loading the entire dataset into memory at once by default.
+        Args:
+            split (str, optional): Defaults to "test".
+            session (TruSession): TruSession instance to persist the dataset.
+            dataset_name (str): Name of the dataset to be persisted - Note this can be different from the standardized BEIR dataset names.
+            dataset_metadata (Optional[Dict[str, Any]], optional): Metadata for the dataset.
+            download (bool, optional): If False, remove the downloaded dataset file after processing. Defaults to True.
+        Returns:
+            pd.DataFrame: DataFrame with the BEIR dataset
+        """
+        for chunk in self._process_dataset(split=split, chunk_size=chunk_size):
+            df_chunk = pd.DataFrame(chunk)
+            session.add_ground_truth_to_dataset(
+                dataset_name=dataset_name,
+                ground_truth_df=df_chunk,
+                dataset_metadata=dataset_metadata,
+            )
+
+        if not download:
+            logger.info(f"Cleaning up downloaded {self.dataset_name} dataset")
+            os.system(
+                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}"
+            )
+            os.system(
+                f"rm -rf {os.path.join(self.data_folder, self.dataset_name)}.zip"
+            )
+
+        logger.info(
+            f"Finished processing dataset {self.dataset_name} in chunks."
+        )