ragas 0.2.2__tar.gz → 0.2.3__tar.gz

{ragas-0.2.2 → ragas-0.2.3}/DEVELOPMENT.md

@@ -21,6 +21,7 @@ This document provides guidelines for developing and contributing to the ragas p
 
 4. **Install Dependencies**
    ```
+   pip install -U setuptools  # Required on newer Python versions (e.g., 3.11)
    pip install -e ".[dev]"
    ```
 
@@ -57,6 +58,11 @@ You can run the following command to check for code style issues:
 make run-ci
 ```
 
+Adding a `V=1` option makes the output more verbose, showing normally hidden commands, like so:
+```bash
+make run-ci V=1
+```
+
 ## Running Tests
 
 To run the test suite:

{ragas-0.2.2 → ragas-0.2.3}/Makefile

@@ -1,56 +1,59 @@
 GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
 
+# Optionally show commands being executed with V=1
+Q := $(if $(V),,@)
+
 help: ## Show all Makefile targets
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+	$(Q)grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: format lint type style clean run-benchmarks
 format: ## Running code formatter: black and isort
 	@echo "(isort) Ordering imports..."
-	@isort .
+	$(Q)isort .
 	@echo "(black) Formatting codebase..."
-	@black --config pyproject.toml src tests docs
+	$(Q)black --config pyproject.toml src tests docs
 	@echo "(black) Formatting stubs..."
-	@find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
+	$(Q)find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
 	@echo "(ruff) Running fix only..."
-	@ruff check src docs tests --fix-only
+	$(Q)ruff check src docs tests --fix-only
 lint: ## Running lint checker: ruff
 	@echo "(ruff) Linting development project..."
-	@ruff check src docs tests
+	$(Q)ruff check src docs tests
 type: ## Running type checker: pyright
 	@echo "(pyright) Typechecking codebase..."
 	PYRIGHT_PYTHON_FORCE_VERSION=latest pyright src/ragas
 clean: ## Clean all generated files
 	@echo "Cleaning all generated files..."
-	@cd $(GIT_ROOT)/docs && make clean
-	@cd $(GIT_ROOT) || exit 1
-	@find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+	$(Q)cd $(GIT_ROOT)/docs && $(MAKE) clean
+	$(Q)cd $(GIT_ROOT) || exit 1
+	$(Q)find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 test: ## Run tests
 	@echo "Running tests..."
-	@pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
+	$(Q)pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
 test-e2e: ## Run end2end tests
 	echo "running end2end tests..."
-	@pytest --nbmake tests/e2e -s
+	$(Q)pytest --nbmake tests/e2e -s
 run-ci: format lint type test ## Running all CI checks
 
 # Docs
 rewrite-docs: ## Use GPT4 to rewrite the documentation
 	@echo "Rewriting the documentation in directory $(DIR)..."
-	@python $(GIT_ROOT)/docs/python alphred.py --directory $(DIR)
+	$(Q)python $(GIT_ROOT)/docs/python alphred.py --directory $(DIR)
 ipynb-to-md: ## Convert ipynb files to md files
-	@python $(GIT_ROOT)/scripts/ipynb_to_md.py
+	$(Q)python $(GIT_ROOT)/scripts/ipynb_to_md.py
 docsite: ## Build and serve documentation
-	@$(MAKE) ipynb-to-md
-	@mkdocs serve
+	$(Q)$(MAKE) ipynb-to-md
+	$(Q)mkdocs serve
 
 # Benchmarks
 run-benchmarks-eval: ## Run benchmarks for Evaluation
 	@echo "Running benchmarks for Evaluation..."
-	@cd $(GIT_ROOT)/tests/benchmarks && python benchmark_eval.py
+	$(Q)cd $(GIT_ROOT)/tests/benchmarks && python benchmark_eval.py
 run-benchmarks-testset: ## Run benchmarks for TestSet Generation
 	@echo "Running benchmarks for TestSet Generation..."
-	@cd $(GIT_ROOT)/tests/benchmarks && python benchmark_testsetgen.py
+	$(Q)cd $(GIT_ROOT)/tests/benchmarks && python benchmark_testsetgen.py
 run-benchmarks-in-docker: ## Run benchmarks in docker
 	@echo "Running benchmarks in docker..."
-	@cd $(GIT_ROOT)
-	docker buildx build --build-arg OPENAI_API_KEY=$(OPENAI_API_KEY) -t ragas-benchmark -f $(GIT_ROOT)/tests/benchmarks/Dockerfile . 
+	$(Q)cd $(GIT_ROOT)
+	docker buildx build --build-arg OPENAI_API_KEY=$(OPENAI_API_KEY) -t ragas-benchmark -f $(GIT_ROOT)/tests/benchmarks/Dockerfile .
 	docker inspect ragas-benchmark:latest | jq ".[0].Size" | numfmt --to=si

{ragas-0.2.2/src/ragas.egg-info → ragas-0.2.3}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: ragas
-Version: 0.2.2
-Description-Content-Type: text/plain
+Version: 0.2.3
+Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: datasets

{ragas-0.2.2 → ragas-0.2.3}/docs/community/index.md

@@ -16,6 +16,9 @@ Meet some of our outstanding members who made significant contributions !
 
 Explore insightful articles, tutorials, and stories written by and for our community members.
 
+- [Luka Panić](https://www.linkedin.com/in/luka-pani%C4%87-20b671277/) shares his work on 
+  - [Ragas Evaluation: In-Depth Insights | PIXION Blog](https://pixion.co/blog/ragas-evaluation-in-depth-insights): A detailed explanation of the metrics and how they are calculated.
+  - [RAG in practice - Test Set Generation | PIXION Blog](https://pixion.co/blog/rag-in-practice-test-set-generation): A tutorial on how to generate a test set using Ragas.
 - [Shanthi Vardhan](https://www.linkedin.com/in/shanthivardhan/) shares how his team at [Atomicwork uses ragas](https://www.atomicwork.com/blog/ragas-improving-atom-accuracy) to improve their AI system's ability to accurately identify and retrieve more precise information for enhanced service management.
 - [Pinecone's](https://pinecone.io/blog) study on how RAGs can enhance capabilities of LLMs in ["RAG makes LLMs better and equal"](https://www.pinecone.io/blog/rag-study/) uses ragas to proves context retrieval makes LLMs provide significantly better results, even when increasing the data size to 1 billion.
 - [Aishwarya Prabhat](https://www.linkedin.com/in/aishwaryaprabhat/) shares her expertise on advanced RAG techniques in her comprehensive guide, ["Performing, Evaluating & Tracking Advanced RAG (ft. AzureML, LlamaIndex & Ragas)"](https://www.linkedin.com/pulse/performing-evaluating-tracking-advanced-rag-ft-azureml-prabhat-i1rkc/).

{ragas-0.2.2 → ragas-0.2.3}/docs/concepts/metrics/available_metrics/agents.md

@@ -71,10 +71,9 @@ scorer = TopicAdherenceScore(mode="recall")
 `ToolCallAccuracy` is a metric that can be used to evaluate the performance of the LLM in identifying and calling the required tools to complete a given task. This metric needs `user_input` and `reference_tool_calls` to evaluate the performance of the LLM in identifying and calling the required tools to complete a given task. The metric is computed by comparing the `reference_tool_calls` with the Tool calls made by the AI. The values range between 0 and 1, with higher values indicating better performance. 
 
 ```python
+from ragas.metrics import ToolCallAccuracy
 from ragas.dataset_schema import  MultiTurnSample
 from ragas.messages import HumanMessage,AIMessage,ToolMessage,ToolCall
-from ragas.metrics import ToolCallAccuracy
-
 
 sample = [
     HumanMessage(content="What's the weather like in New York right now?"),
@@ -89,7 +88,7 @@ sample = [
     AIMessage(content="75°F is approximately 23.9°C.")
 ]
 
-sampl2 = MultiTurnSample(
+sample = MultiTurnSample(
     user_input=sample,
     reference_tool_calls=[
         ToolCall(name="weather_check", args={"location": "New York"}),
@@ -98,7 +97,8 @@ sampl2 = MultiTurnSample(
 )
 
 scorer = ToolCallAccuracy()
-await metric.multi_turn_ascore(sample)
+scorer.llm = your_llm
+await scorer.multi_turn_ascore(sample)
 ```
 
 The tool call sequence specified in `reference_tool_calls` is used as the ideal outcome. If the tool calls made by the AI does not the the order or sequence of the `reference_tool_calls`, the metric will return a score of 0. This helps to ensure that the AI is able to identify and call the required tools in the correct order to complete a given task.
@@ -147,7 +147,8 @@ sample = MultiTurnSample(user_input=[
     reference="Table booked at one of the chinese restaurants at 8 pm")
 
 scorer = AgentGoalAccuracyWithReference()
-await metric.multi_turn_ascore(sample)
+scorer.llm = your_llm
+await scorer.multi_turn_ascore(sample)
 
 ```
 

{ragas-0.2.2 → ragas-0.2.3}/docs/concepts/metrics/available_metrics/index.md

@@ -2,7 +2,7 @@
 
 Ragas provides a set of evaluation metrics that can be used to measure the performance of your LLM application. These metrics are designed to help you objectively measure the performance of your application. Metrics are available for different applications and tasks, such as RAG and Agentic workflows. 
 
-Each metric are essentially paradigms that is designed to evaluate a particular aspect of the application. LLM Based metrics might use one or more LLM calls to arrive at the score or result. One can also modify or write your own metrics using ragas.
+Each metric are essentially paradigms that are designed to evaluate a particular aspect of the application. LLM Based metrics might use one or more LLM calls to arrive at the score or result. One can also modify or write your own metrics using ragas.
 
 ## Retrieval Augmented Generation
 - [Context Precision](context_precision.md)
@@ -11,6 +11,8 @@ Each metric are essentially paradigms that is designed to evaluate a particular
 - [Noise Sensitivity](noise_sensitivity.md)
 - [Response Relevancy](answer_relevance.md)
 - [Faithfulness](faithfulness.md)
+- [Multimodal Faithfulness](multi_modal_faithfulness.md)
+- [Multimodal Relevance](multi_modal_relevance.md)
 
 ## Agents or Tool use cases
 

ragas-0.2.3/docs/concepts/metrics/available_metrics/multi_modal_faithfulness.md

@@ -0,0 +1,50 @@
+## MultiModalFaithfulness
+
+`MultiModalFaithfulness` metric measures the factual consistency of the generated answer against both visual and textual context. It is calculated from the answer, retrieved textual context, and visual context. The answer is scaled to a (0,1) range, with higher scores indicating better faithfulness.
+
+The generated answer is regarded as faithful if all the claims made in the answer can be inferred from either the visual or textual context provided. To determine this, the response is directly evaluated against the provided contexts, and the faithfulness score is either 0 or 1.
+
+### Example
+
+```python
+from ragas.database_schema import SingleTurnSample 
+from ragas.metrics import MultiModalFaithfulness
+
+sample = SingleTurnSample(
+        user_input="What about the Tesla Model X?",
+        response="Cats are cute.",
+        retrieved_contexts=[
+            "custom_eval/multimodal/images/tesla.jpg"
+        ]
+    )
+scorer = MultiModalFaithfulness()
+await scorer.single_turn_ascore(sample)
+```
+
+### How It’s Calculated 
+
+!!! example
+    **Question**: What about the Tesla Model X?
+
+    **Context (visual)**: 
+    - An image of the Tesla Model X (custom_eval/multimodal/images/tesla.jpg)
+
+    **High faithfulness answer**: The Tesla Model X is an electric SUV manufactured by Tesla.
+
+    **Low faithfulness answer**: Cats are cute.
+
+Let's examine how faithfulness was calculated using the low faithfulness answer:
+
+- **Step 1:** Evaluate the generated response against the given contexts.
+    - Response: "Cats are cute."
+
+- **Step 2:** Verify if the response can be inferred from the given context.
+    - Response: No
+
+- **Step 3:** Use the result to determine the faithfulness score.
+
+    $$
+    \text{Faithfulness} = 0
+    $$
+
+In this example, the response "Cats are cute" cannot be inferred from the image of the Tesla Model X, so the faithfulness score is 0.

ragas-0.2.3/docs/concepts/metrics/available_metrics/multi_modal_relevance.md

@@ -0,0 +1,50 @@
+## MultiModalRelevance
+
+`MultiModalRelevance` metric measures the relevance of the generated answer against both visual and textual context. It is calculated from the user input, response, and retrieved contexts (both visual and textual). The answer is scaled to a (0,1) range, with higher scores indicating better relevance.
+
+The generated answer is regarded as relevant if it aligns with the visual or textual context provided. To determine this, the response is directly evaluated against the provided contexts, and the relevance score is either 0 or 1.
+
+### Example
+
+```python
+from ragas.database_schema import SingleTurnSample 
+from ragas.metrics import MultiModalRelevance
+
+sample = SingleTurnSample(
+        user_input="What about the Tesla Model X?",
+        response="Cats are cute.",
+        retrieved_contexts=[
+            "custom_eval/multimodal/images/tesla.jpg"
+        ]
+    )
+scorer = MultiModalRelevance()
+await scorer.single_turn_ascore(sample)
+```
+
+### How It’s Calculated 
+
+!!! example
+    **Question**: What about the Tesla Model X?
+
+    **Context (visual)**: 
+    - An image of the Tesla Model X (custom_eval/multimodal/images/tesla.jpg)
+
+    **High relevance answer**: The Tesla Model X is an electric SUV manufactured by Tesla.
+
+    **Low relevance answer**: Cats are cute.
+
+Let's examine how relevance was calculated using the low relevance answer:
+
+- **Step 1:** Evaluate the generated response against the given contexts.
+    - Response: "Cats are cute."
+
+- **Step 2:** Verify if the response aligns with the given context.
+    - Response: No
+
+- **Step 3:** Use the result to determine the relevance score.
+
+    $$
+    \text{Relevance} = 0
+    $$
+
+In this example, the response "Cats are cute" does not align with the image of the Tesla Model X, so the relevance score is 0.

{ragas-0.2.2 → ragas-0.2.3}/docs/getstarted/rag_testset_generation.md

@@ -99,7 +99,7 @@ Now we will enrich the knowledge graph with additional information using [Transf
 But you can mix and match transforms or build your own as needed.
 
 ```python
-from ragas.testset.transforms import default_transforms
+from ragas.testset.transforms import default_transforms, apply_transforms
 
 
 # define your LLM and Embedding Model

{ragas-0.2.2 → ragas-0.2.3}/docs/howtos/applications/_cost.md

@@ -1,4 +1,4 @@
-# How to estimate Cost and Usage of evaluations
+# How to estimate Cost and Usage of evaluations and testset generation
 
 When using LLMs for evaluation and test set generation, cost will be an important factor. Ragas provides you some tools to help you with that.
 
@@ -34,7 +34,9 @@ get_token_usage_for_openai(llm_result)
 
 You can define your own or import parsers if they are defined. If you would like to suggest parser for LLM providers or contribute your own ones please check out this [issue](https://github.com/explodinggradients/ragas/issues/1151) 🙂.
 
-You can use it for evaluations as so.
+## Token Usage for Evaluations
+
+Let's use the `get_token_usage_for_openai` parser to calculate the token usage for an evaluation.
 
 
 ```python
@@ -43,9 +45,14 @@ from datasets import load_dataset
 
 dataset = load_dataset("explodinggradients/amnesty_qa", "english_v3")
 
-dataset = EvaluationDataset.load_from_hf(dataset["eval"])
+eval_dataset = EvaluationDataset.from_hf_dataset(dataset["eval"])
 ```
 
+    Repo card metadata block was not found. Setting CardData to empty.
+
+
+You can pass in the parser to the `evaluate()` function and the cost will be calculated and returned in the `Result` object.
+
 
 ```python
 from ragas import evaluate
@@ -54,7 +61,7 @@ from ragas.metrics import LLMContextRecall
 from ragas.cost import get_token_usage_for_openai
 
 result = evaluate(
-    amnesty_qa["eval"],
+    eval_dataset,
     metrics=[LLMContextRecall()],
     llm=gpt4o,
     token_usage_parser=get_token_usage_for_openai,
@@ -62,7 +69,7 @@ result = evaluate(
 ```
 
 
-    Evaluating:   0%|          | 0/80 [00:00<?, ?it/s]
+    Evaluating:   0%|          | 0/20 [00:00<?, ?it/s]
 
 
 
@@ -73,7 +80,7 @@ result.total_tokens()
 
 
 
-    TokenUsage(input_tokens=116765, output_tokens=39031, model='')
+    TokenUsage(input_tokens=25097, output_tokens=3757, model='')
 
 
 
@@ -92,3 +99,55 @@ result.total_cost(cost_per_input_token=5 / 1e6, cost_per_output_token=15 / 1e6)
     1.1692900000000002
 
 
+
+## Token Usage for Testset Generation
+
+You can use the same parser for testset generation but you need to pass in the `token_usage_parser` to the `generate()` function. For now it only calculates the cost for the generation process and not the cost for the transforms.
+
+For an example let's load an existing KnowledgeGraph and generate a testset. If you want to know more about how to generate a testset please check out the [testset generation](../../getstarted/rag_testset_generation.md#a-deeper-look).
+
+
+```python
+from ragas.testset.graph import KnowledgeGraph
+
+# loading an existing KnowledgeGraph
+# make sure to change the path to the location of the KnowledgeGraph file
+kg = KnowledgeGraph.load("../../../experiments/scratchpad_kg.json")
+kg
+```
+
+
+
+
+    KnowledgeGraph(nodes: 47, relationships: 109)
+
+
+
+### Choose your LLM
+
+--8<--
+choose_generator_llm.md
+--8<--
+
+
+```python
+from ragas.testset import TestsetGenerator
+from ragas.llms import llm_factory
+
+tg = TestsetGenerator(llm=llm_factory(), knowledge_graph=kg)
+# generating a testset
+testset = tg.generate(testset_size=10, token_usage_parser=get_token_usage_for_openai)
+```
+
+
+```python
+# total cost for the generation process
+testset.total_cost(cost_per_input_token=5 / 1e6, cost_per_output_token=15 / 1e6)
+```
+
+
+
+
+    0.20967000000000002
+
+

{ragas-0.2.2 → ragas-0.2.3}/docs/howtos/applications/cost.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# How to estimate Cost and Usage of evaluations\n",
+    "# How to estimate Cost and Usage of evaluations and testset generation\n",
     "\n",
     "When using LLMs for evaluation and test set generation, cost will be an important factor. Ragas provides you some tools to help you with that."
    ]
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -33,7 +33,7 @@
        "TokenUsage(input_tokens=9, output_tokens=9, model='')"
       ]
      },
-     "execution_count": 1,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -56,39 +56,61 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You can define your own or import parsers if they are defined. If you would like to suggest parser for LLM providers or contribute your own ones please check out this [issue](https://github.com/explodinggradients/ragas/issues/1151) 🙂.\n",
+    "You can define your own or import parsers if they are defined. If you would like to suggest parser for LLM providers or contribute your own ones please check out this [issue](https://github.com/explodinggradients/ragas/issues/1151) 🙂."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Token Usage for Evaluations\n",
     "\n",
-    "You can use it for evaluations as so."
+    "Let's use the `get_token_usage_for_openai` parser to calculate the token usage for an evaluation."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Repo card metadata block was not found. Setting CardData to empty.\n"
+     ]
+    }
+   ],
    "source": [
     "from ragas import EvaluationDataset\n",
     "from datasets import load_dataset\n",
     "\n",
     "dataset = load_dataset(\"explodinggradients/amnesty_qa\", \"english_v3\")\n",
     "\n",
-    "dataset = EvaluationDataset.load_from_hf(dataset[\"eval\"])"
+    "eval_dataset = EvaluationDataset.from_hf_dataset(dataset[\"eval\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can pass in the parser to the `evaluate()` function and the cost will be calculated and returned in the `Result` object."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "40184591e33e4735b3d9905c2b0d6b9e",
+       "model_id": "c9cf15f7bae64320b2bc389b98321a37",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "Evaluating:   0%|          | 0/80 [00:00<?, ?it/s]"
+       "Evaluating:   0%|          | 0/20 [00:00<?, ?it/s]"
       ]
      },
      "metadata": {},
@@ -102,7 +124,7 @@
     "from ragas.cost import get_token_usage_for_openai\n",
     "\n",
     "result = evaluate(\n",
-    "    amnesty_qa[\"eval\"],\n",
+    "    eval_dataset,\n",
     "    metrics=[LLMContextRecall()],\n",
     "    llm=gpt4o,\n",
     "    token_usage_parser=get_token_usage_for_openai,\n",
@@ -111,16 +133,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "TokenUsage(input_tokens=116765, output_tokens=39031, model='')"
+       "TokenUsage(input_tokens=25097, output_tokens=3757, model='')"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -157,6 +179,88 @@
    "source": [
     "result.total_cost(cost_per_input_token=5 / 1e6, cost_per_output_token=15 / 1e6)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Token Usage for Testset Generation\n",
+    "\n",
+    "You can use the same parser for testset generation but you need to pass in the `token_usage_parser` to the `generate()` function. For now it only calculates the cost for the generation process and not the cost for the transforms.\n",
+    "\n",
+    "For an example let's load an existing KnowledgeGraph and generate a testset. If you want to know more about how to generate a testset please check out the [testset generation](../../getstarted/rag_testset_generation.md#a-deeper-look)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "KnowledgeGraph(nodes: 47, relationships: 109)"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from ragas.testset.graph import KnowledgeGraph\n",
+    "\n",
+    "# loading an existing KnowledgeGraph\n",
+    "# make sure to change the path to the location of the KnowledgeGraph file\n",
+    "kg = KnowledgeGraph.load(\"../../../experiments/scratchpad_kg.json\")\n",
+    "kg"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Choose your LLM\n",
+    "\n",
+    "--8<--\n",
+    "choose_generator_llm.md\n",
+    "--8<--"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ragas.testset import TestsetGenerator\n",
+    "from ragas.llms import llm_factory\n",
+    "\n",
+    "tg = TestsetGenerator(llm=llm_factory(), knowledge_graph=kg)\n",
+    "# generating a testset\n",
+    "testset = tg.generate(testset_size=10, token_usage_parser=get_token_usage_for_openai)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.20967000000000002"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# total cost for the generation process\n",
+    "testset.total_cost(cost_per_input_token=5 / 1e6, cost_per_output_token=15 / 1e6)"
+   ]
   }
  ],
  "metadata": {
@@ -175,7 +279,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

{ragas-0.2.2 → ragas-0.2.3}/pyproject.toml

@@ -43,7 +43,7 @@ docs = [
 package-dir = {"" = "src"}
 
 [tool.setuptools.dynamic]
-readme = {file = ["README.md"], content-type = "text/plain"}
+readme = {file = ["README.md"], content-type = "text/markdown"}
 
 [tool.ruff]
 exclude = ["*.ipynb"]

{ragas-0.2.2 → ragas-0.2.3}/src/ragas/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.2.3'
+__version_tuple__ = version_tuple = (0, 2, 3)