ragas 0.2.2__tar.gz → 0.2.4__tar.gz

{ragas-0.2.2 → ragas-0.2.4}/.readthedocs.yml

@@ -7,5 +7,4 @@ build:
   commands:
     - pip install -e .[docs]
     - if [ -n "$GH_TOKEN" ]; then pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git; fi
-    - make ipynb-to-md
     - mkdocs build --site-dir $READTHEDOCS_OUTPUT/html

{ragas-0.2.2 → ragas-0.2.4}/DEVELOPMENT.md

@@ -21,6 +21,7 @@ This document provides guidelines for developing and contributing to the ragas p
 
 4. **Install Dependencies**
    ```
+   pip install -U setuptools  # Required on newer Python versions (e.g., 3.11)
    pip install -e ".[dev]"
    ```
 
@@ -57,6 +58,11 @@ You can run the following command to check for code style issues:
 make run-ci
 ```
 
+Adding a `V=1` option makes the output more verbose, showing normally hidden commands, like so:
+```bash
+make run-ci V=1
+```
+
 ## Running Tests
 
 To run the test suite:

{ragas-0.2.2 → ragas-0.2.4}/Makefile

@@ -1,56 +1,56 @@
 GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
 
+# Optionally show commands being executed with V=1
+Q := $(if $(V),,@)
+
 help: ## Show all Makefile targets
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+	$(Q)grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: format lint type style clean run-benchmarks
 format: ## Running code formatter: black and isort
 	@echo "(isort) Ordering imports..."
-	@isort .
+	$(Q)isort .
 	@echo "(black) Formatting codebase..."
-	@black --config pyproject.toml src tests docs
+	$(Q)black --config pyproject.toml src tests docs
 	@echo "(black) Formatting stubs..."
-	@find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
+	$(Q)find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \;
 	@echo "(ruff) Running fix only..."
-	@ruff check src docs tests --fix-only
+	$(Q)ruff check src docs tests --fix-only
 lint: ## Running lint checker: ruff
 	@echo "(ruff) Linting development project..."
-	@ruff check src docs tests
+	$(Q)ruff check src docs tests
 type: ## Running type checker: pyright
 	@echo "(pyright) Typechecking codebase..."
 	PYRIGHT_PYTHON_FORCE_VERSION=latest pyright src/ragas
 clean: ## Clean all generated files
 	@echo "Cleaning all generated files..."
-	@cd $(GIT_ROOT)/docs && make clean
-	@cd $(GIT_ROOT) || exit 1
-	@find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+	$(Q)cd $(GIT_ROOT)/docs && $(MAKE) clean
+	$(Q)cd $(GIT_ROOT) || exit 1
+	$(Q)find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 test: ## Run tests
 	@echo "Running tests..."
-	@pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
+	$(Q)pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
 test-e2e: ## Run end2end tests
 	echo "running end2end tests..."
-	@pytest --nbmake tests/e2e -s
+	$(Q)pytest --nbmake tests/e2e -s
 run-ci: format lint type test ## Running all CI checks
 
 # Docs
 rewrite-docs: ## Use GPT4 to rewrite the documentation
 	@echo "Rewriting the documentation in directory $(DIR)..."
-	@python $(GIT_ROOT)/docs/python alphred.py --directory $(DIR)
-ipynb-to-md: ## Convert ipynb files to md files
-	@python $(GIT_ROOT)/scripts/ipynb_to_md.py
+	$(Q)python $(GIT_ROOT)/docs/python alphred.py --directory $(DIR)
 docsite: ## Build and serve documentation
-	@$(MAKE) ipynb-to-md
-	@mkdocs serve
+	$(Q)mkdocs serve --dirtyreload
 
 # Benchmarks
 run-benchmarks-eval: ## Run benchmarks for Evaluation
 	@echo "Running benchmarks for Evaluation..."
-	@cd $(GIT_ROOT)/tests/benchmarks && python benchmark_eval.py
+	$(Q)cd $(GIT_ROOT)/tests/benchmarks && python benchmark_eval.py
 run-benchmarks-testset: ## Run benchmarks for TestSet Generation
 	@echo "Running benchmarks for TestSet Generation..."
-	@cd $(GIT_ROOT)/tests/benchmarks && python benchmark_testsetgen.py
+	$(Q)cd $(GIT_ROOT)/tests/benchmarks && python benchmark_testsetgen.py
 run-benchmarks-in-docker: ## Run benchmarks in docker
 	@echo "Running benchmarks in docker..."
-	@cd $(GIT_ROOT)
-	docker buildx build --build-arg OPENAI_API_KEY=$(OPENAI_API_KEY) -t ragas-benchmark -f $(GIT_ROOT)/tests/benchmarks/Dockerfile . 
+	$(Q)cd $(GIT_ROOT)
+	docker buildx build --build-arg OPENAI_API_KEY=$(OPENAI_API_KEY) -t ragas-benchmark -f $(GIT_ROOT)/tests/benchmarks/Dockerfile .
 	docker inspect ragas-benchmark:latest | jq ".[0].Size" | numfmt --to=si

{ragas-0.2.2/src/ragas.egg-info → ragas-0.2.4}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: ragas
-Version: 0.2.2
-Description-Content-Type: text/plain
+Version: 0.2.4
+Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: datasets
@@ -23,10 +23,11 @@ Requires-Dist: rouge_score; extra == "all"
 Requires-Dist: rapidfuzz; extra == "all"
 Requires-Dist: pandas; extra == "all"
 Requires-Dist: datacompy; extra == "all"
+Requires-Dist: llama_index; extra == "all"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.6.1; extra == "docs"
 Requires-Dist: mkdocs-material; extra == "docs"
-Requires-Dist: mkdocs-jupyter; extra == "docs"
+Requires-Dist: mkdocs-material[imaging]; extra == "docs"
 Requires-Dist: mkdocstrings[python]; extra == "docs"
 Requires-Dist: mkdocs-glightbox; extra == "docs"
 Requires-Dist: mkdocs-autorefs; extra == "docs"

{ragas-0.2.2 → ragas-0.2.4}/docs/community/index.md

@@ -16,6 +16,9 @@ Meet some of our outstanding members who made significant contributions !
 
 Explore insightful articles, tutorials, and stories written by and for our community members.
 
+- [Luka Panić](https://www.linkedin.com/in/luka-pani%C4%87-20b671277/) shares his work on 
+    - [Ragas Evaluation: In-Depth Insights | PIXION Blog](https://pixion.co/blog/ragas-evaluation-in-depth-insights): A detailed explanation of the metrics and how they are calculated.
+    - [RAG in practice - Test Set Generation | PIXION Blog](https://pixion.co/blog/rag-in-practice-test-set-generation): A tutorial on how to generate a test set using Ragas.
 - [Shanthi Vardhan](https://www.linkedin.com/in/shanthivardhan/) shares how his team at [Atomicwork uses ragas](https://www.atomicwork.com/blog/ragas-improving-atom-accuracy) to improve their AI system's ability to accurately identify and retrieve more precise information for enhanced service management.
 - [Pinecone's](https://pinecone.io/blog) study on how RAGs can enhance capabilities of LLMs in ["RAG makes LLMs better and equal"](https://www.pinecone.io/blog/rag-study/) uses ragas to proves context retrieval makes LLMs provide significantly better results, even when increasing the data size to 1 billion.
 - [Aishwarya Prabhat](https://www.linkedin.com/in/aishwaryaprabhat/) shares her expertise on advanced RAG techniques in her comprehensive guide, ["Performing, Evaluating & Tracking Advanced RAG (ft. AzureML, LlamaIndex & Ragas)"](https://www.linkedin.com/pulse/performing-evaluating-tracking-advanced-rag-ft-azureml-prabhat-i1rkc/).
@@ -31,13 +34,16 @@ Explore insightful articles, tutorials, and stories written by and for our commu
     - [RAG Evaluation : Computational Metrics in RAG and Calculation Methods in Ragas](https://tech.beatrust.com/entry/2024/05/02/RAG_Evaluation_%3A_Computational_Metrics_in_RAG_and_Calculation_Methods_in_Ragas)
     - [RAG Evaluation: Assessing the Usefulness of Ragas](https://tech.beatrust.com/entry/2024/05/02/RAG_Evaluation%3A_Assessing_the_Usefulness_of_Ragas)
 - [Atita Arora](https://www.linkedin.com/in/atitaarora/) writes about [Evaluating Retrieval Augmented Generation using RAGAS](https://superlinked.com/vectorhub/articles/retrieval-augmented-generation-eval-qdrant-ragas), an end-to-end tutorial on building RAG using [Qdrant](https://qdrant.tech/) and [Langchain](https://www.langchain.com/) and evaluating it with RAGAS. 
-  - *Bonus content* : Learn how to create an evaluation dataset that serves as a reference point for evaluating our RAG pipeline, Understand the RAGAS evaluation metrics and how to make sense of them and putting them in action to test a Naive RAG pipeline and measure its performance using RAGAS metrics. 
-  - *Code walkthrough* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas
-  - *Code walkthrough using [Deepset Haystack](https://haystack.deepset.ai/) and [Mixedbread.ai](https://www.mixedbread.ai/)* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas-haystack
+    - *Bonus content* : Learn how to create an evaluation dataset that serves as a reference point for evaluating our RAG pipeline, Understand the RAGAS evaluation metrics and how to make sense of them and putting them in action to test a Naive RAG pipeline and measure its performance using RAGAS metrics. 
+    - *Code walkthrough* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas
+    - *Code walkthrough using [Deepset Haystack](https://haystack.deepset.ai/) and [Mixedbread.ai](https://www.mixedbread.ai/)* : https://github.com/qdrant/qdrant-rag-eval/tree/master/workshop-rag-eval-qdrant-ragas-haystack
 - [Minoru Onda](https://x.com/minorun365) writes for beginners about how to start Ragas v0.2 evaluation with Amazon Bedrock, and integrate with Langfuse.
-  - [RAG精度評価の定番ツール「Ragas」にAWSのBedrockで入門しよう！（v0.2対応） - Qiita](https://qiita.com/minorun365/items/2f4e238f8bbc6e393ba5)
-  - [生成AIアプリの出力をRagasで評価して、LangfuseでGUI監視しよう！ - Qiita](https://qiita.com/minorun365/items/70ad2f5a0afaac6e5cb9)
-
+    - [RAG精度評価の定番ツール「Ragas」にAWSのBedrockで入門しよう！（v0.2対応） - Qiita](https://qiita.com/minorun365/items/2f4e238f8bbc6e393ba5)
+    - [生成AIアプリの出力をRagasで評価して、LangfuseでGUI監視しよう！ - Qiita](https://qiita.com/minorun365/items/70ad2f5a0afaac6e5cb9)
+- [Yunnglin](https://github.com/Yunnglin) has penned a guide on integrating Ragas v0.2 into [EvalScope](https://github.com/modelscope/eval-scope) (an evaluation framework for large models), thereby utilizing the [ModelScope](https://github.com/modelscope/modelscope) ecosystem.
+    - Tutorial: [Using Ragas with EvalScope](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html)
+    - 教程: [在EvalScope中使用Ragas一键发起RAG评估](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/backend/rageval_backend/ragas.html)
+    - 最佳实践: [打破文本边界：如何进行多模态RAG评估](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag)
 
 ## **📅 Events**
 

{ragas-0.2.2 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/agents.md

@@ -71,10 +71,9 @@ scorer = TopicAdherenceScore(mode="recall")
 `ToolCallAccuracy` is a metric that can be used to evaluate the performance of the LLM in identifying and calling the required tools to complete a given task. This metric needs `user_input` and `reference_tool_calls` to evaluate the performance of the LLM in identifying and calling the required tools to complete a given task. The metric is computed by comparing the `reference_tool_calls` with the Tool calls made by the AI. The values range between 0 and 1, with higher values indicating better performance. 
 
 ```python
+from ragas.metrics import ToolCallAccuracy
 from ragas.dataset_schema import  MultiTurnSample
 from ragas.messages import HumanMessage,AIMessage,ToolMessage,ToolCall
-from ragas.metrics import ToolCallAccuracy
-
 
 sample = [
     HumanMessage(content="What's the weather like in New York right now?"),
@@ -89,7 +88,7 @@ sample = [
     AIMessage(content="75°F is approximately 23.9°C.")
 ]
 
-sampl2 = MultiTurnSample(
+sample = MultiTurnSample(
     user_input=sample,
     reference_tool_calls=[
         ToolCall(name="weather_check", args={"location": "New York"}),
@@ -98,7 +97,8 @@ sampl2 = MultiTurnSample(
 )
 
 scorer = ToolCallAccuracy()
-await metric.multi_turn_ascore(sample)
+scorer.llm = your_llm
+await scorer.multi_turn_ascore(sample)
 ```
 
 The tool call sequence specified in `reference_tool_calls` is used as the ideal outcome. If the tool calls made by the AI does not the the order or sequence of the `reference_tool_calls`, the metric will return a score of 0. This helps to ensure that the AI is able to identify and call the required tools in the correct order to complete a given task.
@@ -147,7 +147,8 @@ sample = MultiTurnSample(user_input=[
     reference="Table booked at one of the chinese restaurants at 8 pm")
 
 scorer = AgentGoalAccuracyWithReference()
-await metric.multi_turn_ascore(sample)
+scorer.llm = your_llm
+await scorer.multi_turn_ascore(sample)
 
 ```
 

{ragas-0.2.2 → ragas-0.2.4}/docs/concepts/metrics/available_metrics/index.md

@@ -2,7 +2,7 @@
 
 Ragas provides a set of evaluation metrics that can be used to measure the performance of your LLM application. These metrics are designed to help you objectively measure the performance of your application. Metrics are available for different applications and tasks, such as RAG and Agentic workflows. 
 
-Each metric are essentially paradigms that is designed to evaluate a particular aspect of the application. LLM Based metrics might use one or more LLM calls to arrive at the score or result. One can also modify or write your own metrics using ragas.
+Each metric are essentially paradigms that are designed to evaluate a particular aspect of the application. LLM Based metrics might use one or more LLM calls to arrive at the score or result. One can also modify or write your own metrics using ragas.
 
 ## Retrieval Augmented Generation
 - [Context Precision](context_precision.md)
@@ -11,6 +11,8 @@ Each metric are essentially paradigms that is designed to evaluate a particular
 - [Noise Sensitivity](noise_sensitivity.md)
 - [Response Relevancy](answer_relevance.md)
 - [Faithfulness](faithfulness.md)
+- [Multimodal Faithfulness](multi_modal_faithfulness.md)
+- [Multimodal Relevance](multi_modal_relevance.md)
 
 ## Agents or Tool use cases
 

ragas-0.2.4/docs/concepts/metrics/available_metrics/multi_modal_faithfulness.md

@@ -0,0 +1,50 @@
+## MultiModalFaithfulness
+
+`MultiModalFaithfulness` metric measures the factual consistency of the generated answer against both visual and textual context. It is calculated from the answer, retrieved textual context, and visual context. The answer is scaled to a (0,1) range, with higher scores indicating better faithfulness.
+
+The generated answer is regarded as faithful if all the claims made in the answer can be inferred from either the visual or textual context provided. To determine this, the response is directly evaluated against the provided contexts, and the faithfulness score is either 0 or 1.
+
+### Example
+
+```python
+from ragas.database_schema import SingleTurnSample 
+from ragas.metrics import MultiModalFaithfulness
+
+sample = SingleTurnSample(
+        user_input="What about the Tesla Model X?",
+        response="Cats are cute.",
+        retrieved_contexts=[
+            "custom_eval/multimodal/images/tesla.jpg"
+        ]
+    )
+scorer = MultiModalFaithfulness()
+await scorer.single_turn_ascore(sample)
+```
+
+### How It’s Calculated 
+
+!!! example
+    **Question**: What about the Tesla Model X?
+
+    **Context (visual)**: 
+    - An image of the Tesla Model X (custom_eval/multimodal/images/tesla.jpg)
+
+    **High faithfulness answer**: The Tesla Model X is an electric SUV manufactured by Tesla.
+
+    **Low faithfulness answer**: Cats are cute.
+
+Let's examine how faithfulness was calculated using the low faithfulness answer:
+
+- **Step 1:** Evaluate the generated response against the given contexts.
+    - Response: "Cats are cute."
+
+- **Step 2:** Verify if the response can be inferred from the given context.
+    - Response: No
+
+- **Step 3:** Use the result to determine the faithfulness score.
+
+    $$
+    \text{Faithfulness} = 0
+    $$
+
+In this example, the response "Cats are cute" cannot be inferred from the image of the Tesla Model X, so the faithfulness score is 0.

ragas-0.2.4/docs/concepts/metrics/available_metrics/multi_modal_relevance.md

@@ -0,0 +1,50 @@
+## MultiModalRelevance
+
+`MultiModalRelevance` metric measures the relevance of the generated answer against both visual and textual context. It is calculated from the user input, response, and retrieved contexts (both visual and textual). The answer is scaled to a (0,1) range, with higher scores indicating better relevance.
+
+The generated answer is regarded as relevant if it aligns with the visual or textual context provided. To determine this, the response is directly evaluated against the provided contexts, and the relevance score is either 0 or 1.
+
+### Example
+
+```python
+from ragas.database_schema import SingleTurnSample 
+from ragas.metrics import MultiModalRelevance
+
+sample = SingleTurnSample(
+        user_input="What about the Tesla Model X?",
+        response="Cats are cute.",
+        retrieved_contexts=[
+            "custom_eval/multimodal/images/tesla.jpg"
+        ]
+    )
+scorer = MultiModalRelevance()
+await scorer.single_turn_ascore(sample)
+```
+
+### How It’s Calculated 
+
+!!! example
+    **Question**: What about the Tesla Model X?
+
+    **Context (visual)**: 
+    - An image of the Tesla Model X (custom_eval/multimodal/images/tesla.jpg)
+
+    **High relevance answer**: The Tesla Model X is an electric SUV manufactured by Tesla.
+
+    **Low relevance answer**: Cats are cute.
+
+Let's examine how relevance was calculated using the low relevance answer:
+
+- **Step 1:** Evaluate the generated response against the given contexts.
+    - Response: "Cats are cute."
+
+- **Step 2:** Verify if the response aligns with the given context.
+    - Response: No
+
+- **Step 3:** Use the result to determine the relevance score.
+
+    $$
+    \text{Relevance} = 0
+    $$
+
+In this example, the response "Cats are cute" does not align with the image of the Tesla Model X, so the relevance score is 0.

ragas-0.2.4/docs/extra/components/choose_evaluator_llm.md

@@ -0,0 +1,140 @@
+
+=== "OpenAI"
+    Install the langchain-openai package
+
+    ```bash
+    pip install langchain-openai
+    ```
+
+    Ensure you have your OpenAI key ready and available in your environment.
+
+    ```python
+    import os
+    os.environ["OPENAI_API_KEY"] = "your-openai-key"
+    ```
+    Wrap the LLMs in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from ragas.llms import LangchainLLMWrapper
+    from ragas.embeddings import LangchainEmbeddingsWrapper
+    from langchain_openai import ChatOpenAI
+    from langchain_openai import OpenAIEmbeddings
+    evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
+    evaluator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
+    ```
+
+
+=== "Amazon Bedrock"
+    Install the langchain-aws package
+
+    ```bash
+    pip install langchain-aws
+    ```
+
+    Then you have to set your AWS credentials and configurations
+
+    ```python
+    config = {
+        "credentials_profile_name": "your-profile-name",  # E.g "default"
+        "region_name": "your-region-name",  # E.g. "us-east-1"
+        "llm": "your-llm-model-id",  # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
+        "embeddings": "your-embedding-model-id",  # E.g "amazon.titan-embed-text-v2:0"
+        "temperature": 0.4,
+    }
+    ```
+
+    Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from langchain_aws import ChatBedrockConverse
+    from langchain_aws import BedrockEmbeddings
+    from ragas.llms import LangchainLLMWrapper
+    from ragas.embeddings import LangchainEmbeddingsWrapper
+
+    evaluator_llm = LangchainLLMWrapper(ChatBedrockConverse(
+        credentials_profile_name=config["credentials_profile_name"],
+        region_name=config["region_name"],
+        base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com",
+        model=config["llm"],
+        temperature=config["temperature"],
+    ))
+    evaluator_embeddings = LangchainEmbeddingsWrapper(BedrockEmbeddings(
+        credentials_profile_name=config["credentials_profile_name"],
+        region_name=config["region_name"],
+        model_id=config["embeddings"],
+    ))
+    ```
+
+    If you want more information on how to use other AWS services, please refer to the [langchain-aws](https://python.langchain.com/docs/integrations/providers/aws/) documentation.
+
+=== "Azure OpenAI"
+    Install the langchain-openai package
+
+    ```bash
+    pip install langchain-openai
+    ```
+
+    Ensure you have your Azure OpenAI key ready and available in your environment.
+
+    ```python
+    import os
+    os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-openai-key"
+
+    # other configuration
+    azure_config = {
+        "base_url": "",  # your endpoint
+        "model_deployment": "",  # your model deployment name
+        "model_name": "",  # your model name
+        "embedding_deployment": "",  # your embedding deployment name
+        "embedding_name": "",  # your embedding name
+    }
+
+    ```
+
+    Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from langchain_openai import AzureChatOpenAI
+    from langchain_openai import AzureOpenAIEmbeddings
+    from ragas.llms import LangchainLLMWrapper
+    from ragas.embeddings import LangchainEmbeddingsWrapper
+    evaluator_llm = LangchainLLMWrapper(AzureChatOpenAI(
+        openai_api_version="2023-05-15",
+        azure_endpoint=azure_configs["base_url"],
+        azure_deployment=azure_configs["model_deployment"],
+        model=azure_configs["model_name"],
+        validate_base_url=False,
+    ))
+
+    # init the embeddings for answer_relevancy, answer_correctness and answer_similarity
+    evaluator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(
+        openai_api_version="2023-05-15",
+        azure_endpoint=azure_configs["base_url"],
+        azure_deployment=azure_configs["embedding_deployment"],
+        model=azure_configs["embedding_name"],
+    ))
+    ```
+
+    If you want more information on how to use other Azure services, please refer to the [langchain-azure](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) documentation.
+
+
+=== "Others"
+    If you are using a different LLM provider and using Langchain to interact with it, you can wrap your LLM in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from ragas.llms import LangchainLLMWrapper
+    evaluator_llm = LangchainLLMWrapper(your_llm_instance)
+    ```
+
+    For a more detailed guide, checkout [the guide on customizing models](../../howtos/customizations/customize_models/).
+
+    If you using LlamaIndex, you can use the `LlamaIndexLLMWrapper` to wrap your LLM so that it can be used with ragas.
+
+    ```python
+    from ragas.llms import LlamaIndexLLMWrapper
+    evaluator_llm = LlamaIndexLLMWrapper(your_llm_instance)
+    ```
+
+    For more information on how to use LlamaIndex, please refer to the [LlamaIndex Integration guide](../../howtos/integrations/_llamaindex/).
+
+    If your still not able use Ragas with your favorite LLM provider, please let us know by by commenting on this [issue](https://github.com/explodinggradients/ragas/issues/1617) and we'll add support for it 🙂.

ragas-0.2.4/docs/extra/components/choose_generator_llm.md

@@ -0,0 +1,138 @@
+=== "OpenAI"
+    Install the langchain-openai package
+
+    ```bash
+    pip install langchain-openai
+    ```
+
+    then ensure you have your OpenAI key ready and available in your environment
+
+    ```python
+    import os
+    os.environ["OPENAI_API_KEY"] = "your-openai-key"
+    ```
+
+    Wrapp the LLMs in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from ragas.llms import LangchainLLMWrapper
+    from langchain_openai import ChatOpenAI
+    from langchain_openai import OpenAIEmbeddings
+    generator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o"))
+    generator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
+    ```
+
+
+=== "Amazon Bedrock"
+    Install the langchain-aws package
+
+    ```bash
+    pip install langchain-aws
+    ```
+
+    then you have to set your AWS credentials and configurations
+
+    ```python
+    config = {
+        "credentials_profile_name": "your-profile-name",  # E.g "default"
+        "region_name": "your-region-name",  # E.g. "us-east-1"
+        "llm": "your-llm-model-id",  # E.g "anthropic.claude-3-5-sonnet-20241022-v2:0"
+        "embeddings": "your-embedding-model-id",  # E.g "amazon.titan-embed-text-v2:0"
+        "temperature": 0.4,
+    }
+    ```
+
+    define you LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from langchain_aws import ChatBedrockConverse
+    from langchain_aws import BedrockEmbeddings
+    from ragas.llms import LangchainLLMWrapper
+    from ragas.embeddings import LangchainEmbeddingsWrapper
+
+    generator_llm = LangchainLLMWrapper(ChatBedrockConverse(
+        credentials_profile_name=config["credentials_profile_name"],
+        region_name=config["region_name"],
+        base_url=f"https://bedrock-runtime.{config['region_name']}.amazonaws.com",
+        model=config["llm"],
+        temperature=config["temperature"],
+    ))
+    generator_embeddings = LangchainEmbeddingsWrapper(BedrockEmbeddings(
+        credentials_profile_name=config["credentials_profile_name"],
+        region_name=config["region_name"],
+        model_id=config["embeddings"],
+    ))
+    ```
+
+    If you want more information on how to use other AWS services, please refer to the [langchain-aws](https://python.langchain.com/docs/integrations/providers/aws/) documentation.
+
+=== "Azure OpenAI"
+    Install the langchain-openai package
+
+    ```bash
+    pip install langchain-openai
+    ```
+
+    Ensure you have your Azure OpenAI key ready and available in your environment.
+
+    ```python
+    import os
+    os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-openai-key"
+
+    # other configuration
+    azure_config = {
+        "base_url": "",  # your endpoint
+        "model_deployment": "",  # your model deployment name
+        "model_name": "",  # your model name
+        "embedding_deployment": "",  # your embedding deployment name
+        "embedding_name": "",  # your embedding name
+    }
+
+    ```
+
+    Define your LLMs and wrap them in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from langchain_openai import AzureChatOpenAI
+    from langchain_openai import AzureOpenAIEmbeddings
+    from ragas.llms import LangchainLLMWrapper
+    from ragas.embeddings import LangchainEmbeddingsWrapper
+    generator_llm = LangchainLLMWrapper(AzureChatOpenAI(
+        openai_api_version="2023-05-15",
+        azure_endpoint=azure_configs["base_url"],
+        azure_deployment=azure_configs["model_deployment"],
+        model=azure_configs["model_name"],
+        validate_base_url=False,
+    ))
+
+    # init the embeddings for answer_relevancy, answer_correctness and answer_similarity
+    generator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(
+        openai_api_version="2023-05-15",
+        azure_endpoint=azure_configs["base_url"],
+        azure_deployment=azure_configs["embedding_deployment"],
+        model=azure_configs["embedding_name"],
+    ))
+    ```
+
+    If you want more information on how to use other Azure services, please refer to the [langchain-azure](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) documentation.
+
+=== "Others"
+    If you are using a different LLM provider and using Langchain to interact with it, you can wrap your LLM in `LangchainLLMWrapper` so that it can be used with ragas.
+
+    ```python
+    from ragas.llms import LangchainLLMWrapper
+    generator_llm = LangchainLLMWrapper(your_llm_instance)
+    ```
+
+    For a more detailed guide, checkout [the guide on customizing models](../../howtos/customizations/customize_models/).
+
+    If you using LlamaIndex, you can use the `LlamaIndexLLMWrapper` to wrap your LLM so that it can be used with ragas.
+
+    ```python
+    from ragas.llms import LlamaIndexLLMWrapper
+    generator_llm = LlamaIndexLLMWrapper(your_llm_instance)
+    ```
+
+    For more information on how to use LlamaIndex, please refer to the [LlamaIndex Integration guide](../../howtos/integrations/_llamaindex/).
+
+    If your still not able use Ragas with your favorite LLM provider, please let us know by by commenting on this [issue](https://github.com/explodinggradients/ragas/issues/1617) and we'll add support for it 🙂.

{ragas-0.2.2 → ragas-0.2.4}/docs/getstarted/rag_testset_generation.md

@@ -39,7 +39,7 @@ Now we will run the test generation using the loaded documents and the LLM setup
 ```python
 from ragas.testset import TestsetGenerator
 
-generator = TestsetGenerator(llm=generator_llm)
+generator = TestsetGenerator(llm=generator_llm, embedding_model=generator_embeddings)
 dataset = generator.generate_with_langchain_docs(docs, testset_size=10)
 ```
 
@@ -99,7 +99,7 @@ Now we will enrich the knowledge graph with additional information using [Transf
 But you can mix and match transforms or build your own as needed.
 
 ```python
-from ragas.testset.transforms import default_transforms
+from ragas.testset.transforms import default_transforms, apply_transforms
 
 
 # define your LLM and Embedding Model
@@ -141,9 +141,9 @@ query_distribution = default_query_distribution(generator_llm)
 ```
 ```
 [
-    (AbstractQuerySynthesizer(llm=generator_llm), 0.25),
-    (ComparativeAbstractQuerySynthesizer(llm=generator_llm), 0.25),
-    (SpecificQuerySynthesizer(llm=generator_llm), 0.5),
+        (SingleHopSpecificQuerySynthesizer(llm=llm), 0.5),
+        (MultiHopAbstractQuerySynthesizer(llm=llm), 0.25),
+        (MultiHopSpecificQuerySynthesizer(llm=llm), 0.25),
 ]
 ```