AutoRAG 0.3.2__tar.gz → 0.3.3__tar.gz

{autorag-0.3.2 → autorag-0.3.3}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.2
+Version: 0.3.3
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -238,14 +238,12 @@ Requires-Dist: click
 Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
-Requires-Dist: guidance
 Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score
 Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
-Requires-Dist: ragas
 Requires-Dist: llmlingua
 Requires-Dist: peft
 Requires-Dist: llama-index>=0.11.0
@@ -263,6 +261,7 @@ Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.3.0
 Requires-Dist: langchain-unstructured>=0.1.5
 Requires-Dist: langchain-upstage
+Requires-Dist: langchain-community>=0.3.0
 Requires-Dist: panel
 Requires-Dist: seaborn
 Requires-Dist: ipykernel

{autorag-0.3.2 → autorag-0.3.3}/AutoRAG.egg-info/SOURCES.txt

@@ -87,6 +87,7 @@ autorag/evaluation/retrieval.py
 autorag/evaluation/retrieval_contents.py
 autorag/evaluation/util.py
 autorag/evaluation/metric/__init__.py
+autorag/evaluation/metric/deepeval_prompt.py
 autorag/evaluation/metric/generation.py
 autorag/evaluation/metric/retrieval.py
 autorag/evaluation/metric/retrieval_contents.py

{autorag-0.3.2 → autorag-0.3.3}/AutoRAG.egg-info/requires.txt

@@ -17,14 +17,12 @@ click
 Flask
 torch
 sentencepiece
-guidance
 cohere>=5.8.0
 tokenlog>=0.0.2
 aiohttp
 bert_score
 sentence-transformers
 FlagEmbedding
-ragas
 llmlingua
 peft
 llama-index>=0.11.0
@@ -42,6 +40,7 @@ gradio
 langchain-core>=0.3.0
 langchain-unstructured>=0.1.5
 langchain-upstage
+langchain-community>=0.3.0
 panel
 seaborn
 ipykernel

{autorag-0.3.2 → autorag-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.2
+Version: 0.3.3
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -238,14 +238,12 @@ Requires-Dist: click
 Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
-Requires-Dist: guidance
 Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score
 Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
-Requires-Dist: ragas
 Requires-Dist: llmlingua
 Requires-Dist: peft
 Requires-Dist: llama-index>=0.11.0
@@ -263,6 +261,7 @@ Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.3.0
 Requires-Dist: langchain-unstructured>=0.1.5
 Requires-Dist: langchain-upstage
+Requires-Dist: langchain-community>=0.3.0
 Requires-Dist: panel
 Requires-Dist: seaborn
 Requires-Dist: ipykernel

autorag-0.3.3/autorag/VERSION

@@ -0,0 +1 @@
+0.3.3

{autorag-0.3.2 → autorag-0.3.3}/autorag/data/legacy/qacreation/ragas.py

@@ -5,8 +5,6 @@ import pandas as pd
 from langchain_core.embeddings import Embeddings
 from langchain_core.language_models import BaseChatModel
 from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-from ragas.testset import TestsetGenerator
-from ragas.testset.evolutions import simple, reasoning, multi_context
 
 from autorag.data.utils.util import corpus_df_to_langchain_documents
 from autorag.utils import cast_qa_dataset
@@ -37,6 +35,9 @@ def generate_qa_ragas(
 	    You can input 'with_debugging_logs', 'is_async', 'raise_exceptions', and 'run_config'.
 	:return: QA dataset dataframe.
 	"""
+	from ragas.testset import TestsetGenerator
+	from ragas.testset.evolutions import simple, reasoning, multi_context
+
 	if generator_llm is None:
 		generator_llm = ChatOpenAI(model="gpt-3.5-turbo-16k")
 	if critic_llm is None:

{autorag-0.3.2 → autorag-0.3.3}/autorag/data/legacy/qacreation/simple.py

@@ -3,12 +3,10 @@ import pathlib
 import uuid
 from typing import Callable
 
-import guidance
 import pandas as pd
-from guidance import models, gen
 
 
-def generate_qa_row(llm: models.Model, corpus_data_row):
+def generate_qa_row(llm, corpus_data_row):
 	"""
 	this sample code to generate rag dataset using OpenAI chat model
 
@@ -16,6 +14,9 @@ def generate_qa_row(llm: models.Model, corpus_data_row):
 	:param corpus_data_row: need "contents" column
 	:return: should to be dict which has "query", "generation_gt" columns at least.
 	"""
+	from guidance import gen
+	import guidance
+
 	temp_llm = llm
 	with guidance.user():
 		temp_llm += f"""
@@ -48,7 +49,7 @@ def generate_qa_row(llm: models.Model, corpus_data_row):
 
 
 def generate_simple_qa_dataset(
-	llm: models.Model,
+	llm,
 	corpus_data: pd.DataFrame,
 	output_filepath: str,
 	generate_row_function: Callable,

{autorag-0.3.2 → autorag-0.3.3}/autorag/data/parse/clova.py

@@ -99,8 +99,10 @@ async def clova_ocr_pure(
 			raise RuntimeError(
 				f"Invalid response from Clova API: {resp_json['detail']}"
 			)
-
-		table_html = json_to_html_table(resp_json["images"][0]["tables"][0]["cells"])
+		if "tables" in resp_json["images"][0].keys():
+			table_html = json_to_html_table(
+				resp_json["images"][0]["tables"][0]["cells"]
+			)
 		page_text = extract_text_from_fields(resp_json["images"][0]["fields"])
 
 		if table_html:

{autorag-0.3.2 → autorag-0.3.3}/autorag/data/parse/langchain_parse.py

@@ -1,4 +1,5 @@
 import multiprocessing as mp
+from itertools import chain
 from typing import List, Tuple
 
 from autorag.data import parse_modules
@@ -20,6 +21,7 @@ def langchain_parse(
 	if parse_method in ["directory", "unstructured"]:
 		results = parse_all_files(data_path_list, parse_method, **kwargs)
 		texts, path = results[0], results[1]
+		pages = [-1] * len(texts)
 
 	else:
 		num_workers = mp.cpu_count()
@@ -30,14 +32,14 @@ def langchain_parse(
 				[(data_path, parse_method, kwargs) for data_path in data_path_list],
 			)
 
-		texts, path = zip(*results)
-		texts, path = list(texts), list(path)
-	pages = [-1] * len(texts)
+		texts, path, pages = (list(chain.from_iterable(item)) for item in zip(*results))
 
 	return texts, path, pages
 
 
-def langchain_parse_pure(data_path: str, parse_method: str, kwargs) -> Tuple[str, str]:
+def langchain_parse_pure(
+	data_path: str, parse_method: str, kwargs
+) -> Tuple[List[str], List[str], List[int]]:
 	"""
 	Parses a single file using the specified parse method.
 
@@ -54,12 +56,18 @@ def langchain_parse_pure(data_path: str, parse_method: str, kwargs) -> Tuple[str
 
 	# Load the text from the file
 	documents = parse_instance.load()
-	text = documents[0].page_content
+
+	texts = list(map(lambda x: x.page_content, documents))
+	path = [data_path] * len(texts)
+	if parse_method in ["pymupdf", "pdfplumber", "pypdf", "pypdfium2"]:
+		pages = list(range(1, len(documents) + 1))
+	else:
+		pages = [-1] * len(texts)
 
 	# Clean up the parse instance
 	del parse_instance
 
-	return text, data_path
+	return texts, path, pages
 
 
 def parse_all_files(

{autorag-0.3.2 → autorag-0.3.3}/autorag/data/parse/llamaparse.py

@@ -1,4 +1,5 @@
 from typing import List, Tuple
+from itertools import chain
 
 from llama_parse import LlamaParse
 
@@ -30,15 +31,18 @@ def llama_parse(
 
 	del parse_instance
 
-	texts, path = zip(*results)
-	pages = [-1] * len(texts)
+	texts, path, pages = (list(chain.from_iterable(item)) for item in zip(*results))
 
-	return list(texts), list(path), pages
+	return texts, path, pages
 
 
-async def llama_parse_pure(data_path: str, parse_instance) -> Tuple[str, str]:
+async def llama_parse_pure(
+	data_path: str, parse_instance
+) -> Tuple[List[str], List[str], List[int]]:
 	documents = await parse_instance.aload_data(data_path)
 
-	text = documents[0].text
+	texts = list(map(lambda x: x.text, documents))
+	path = [data_path] * len(texts)
+	pages = list(range(1, len(documents) + 1))
 
-	return text, data_path
+	return texts, path, pages

{autorag-0.3.2 → autorag-0.3.3}/autorag/evaluation/generation.py

@@ -11,17 +11,27 @@ from autorag.evaluation.metric.generation import (
 	sem_score,
 	g_eval,
 	bert_score,
+	deepeval_faithfulness,
 )
 from autorag.evaluation.util import cast_metrics
 from autorag.schema.metricinput import MetricInput
 
 GENERATION_METRIC_FUNC_DICT = {
-	func.__name__: func for func in [bleu, meteor, rouge, sem_score, g_eval, bert_score]
+	func.__name__: func
+	for func in [
+		bleu,
+		meteor,
+		rouge,
+		sem_score,
+		g_eval,
+		bert_score,
+		deepeval_faithfulness,
+	]
 }
 
 
 def evaluate_generation(
-		metric_inputs: List[MetricInput], metrics: Union[List[str], List[Dict]]
+	metric_inputs: List[MetricInput], metrics: Union[List[str], List[Dict]]
 ):
 	def decorator_evaluate_generation(func: Callable):
 		@functools.wraps(func)
@@ -55,7 +65,6 @@ def evaluate_generation(
 						f"{metric_name} will be ignored."
 					)
 				else:
-
 					metric_scores[metric_name] = GENERATION_METRIC_FUNC_DICT[
 						metric_name
 					](

{autorag-0.3.2 → autorag-0.3.3}/autorag/evaluation/metric/__init__.py

@@ -1,4 +1,12 @@
-from .generation import bleu, meteor, rouge, sem_score, g_eval, bert_score
+from .generation import (
+	bleu,
+	meteor,
+	rouge,
+	sem_score,
+	g_eval,
+	bert_score,
+	deepeval_faithfulness,
+)
 from .retrieval import (
 	retrieval_f1,
 	retrieval_recall,

autorag-0.3.3/autorag/evaluation/metric/deepeval_prompt.py

@@ -0,0 +1,218 @@
+class FaithfulnessTemplate:
+	@staticmethod
+	def generate_claims(text, lang: str = "en"):
+		if lang == "en":
+			return f"""Based on the given text, please generate a comprehensive list of FACTUAL claims that can inferred from the provided text.
+
+	Example:
+	Example Text:
+	"Einstein won the noble prize in 1968 for his discovery of the photoelectric effect."
+
+	Example JSON:
+	{{
+	    "claims": [
+	        "Einstein won the noble prize for his discovery of the photoelectric effect.",
+	        "Einstein won the noble prize in 1968."
+	    ]
+	}}
+	===== END OF EXAMPLE ======
+
+	**
+	IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
+	Only include claims that are factual, and the claims you extract should include the full context it was presented in, NOT cherry picked facts.
+	You should NOT include any prior knowledge, and take the text at face value when extracting claims.
+	**
+
+	Text:
+	{text}
+
+	JSON:
+	"""
+		elif lang == "ko":
+			return f"""주어진 텍스트에서 찾을 수 있는 사실적 정보들의 목록을 생성하세요.
+
+예시:
+예시 텍스트:
+“아인슈타인은 1968년에 광전 효과 발견으로 노벨상을 수상했다.”
+
+예시 JSON:
+{{
+“claims”: [
+“아인슈타인은 광전 효과 발견으로 노벨상을 수상했다.”,
+“아인슈타인은 1968년에 노벨상을 수상했다.”
+]
+}}
+===== 예시 끝 ======
+
+**
+중요: 오직 JSON 형식으로 “claims” 키가 문자열 목록으로 반환되도록 해야 합니다. 다른 단어나 설명은 필요하지 않습니다.
+사실에 기반한 주장만 포함하며, 추출한 주장은 전체 맥락을 유지해야 하며, 부분적으로 선택된 사실을 포함하지 않아야 합니다.
+사전 지식은 포함하지 말고, 텍스트에만 기초해 주장들을 추출해야 합니다.
+**
+
+텍스트:
+{text}
+
+JSON:
+"""
+		else:
+			raise ValueError(f"Language {lang} is not supported.")
+
+	@staticmethod
+	def generate_truths(text, lang: str = "en"):
+		if lang == "en":
+			return f"""Based on the given text, please generate a comprehensive list of FACTUAL, undisputed truths that can inferred from the provided text.
+
+	Example:
+	Example Text:
+	"Einstein won the noble prize in 1968 for his discovery of the photoelectric effect."
+
+	Example JSON:
+	{{
+	    "truths": [
+	        "Einstein won the noble prize for his discovery of the photoelectric effect.",
+	        "Einstein won the noble prize in 1968."
+	    ]
+	}}
+	===== END OF EXAMPLE ======
+
+	**
+	IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
+	Only include truths that are factual.
+	**
+
+	Text:
+	{text}
+
+	JSON:
+	"""
+		elif lang == "ko":
+			return f"""주어진 텍스트에서 추출할 수 있는 사실적이고 논란이 없는 진실들의 목록을 생성하세요.
+
+예시:
+예시 텍스트:
+"아인슈타인은 1968년에 광전 효과 발견으로 노벨상을 수상했다."
+
+예시 JSON:
+{{
+    "truths": [
+        "아인슈타인은 광전 효과 발견으로 노벨상을 수상했다.",
+        "아인슈타인은 1968년에 노벨상을 수상했다."
+    ]
+}}
+===== 예시 끝 ======
+
+**
+중요: 오직 JSON 형식으로 "truths" 키가 문자열 목록으로 반환되도록 해야 합니다. 다른 단어나 설명은 필요하지 않습니다.
+사실에 기반한 진실만 포함해야 합니다.
+**
+
+텍스트:
+{text}
+
+JSON:"""
+		else:
+			raise ValueError(f"Language {lang} is not supported.")
+
+	@staticmethod
+	def generate_verdicts(claims, retrieval_context, lang: str = "en"):
+		if lang == "en":
+			return f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
+	The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context.
+	Provide a 'reason' ONLY if the answer is 'no'.
+	The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
+
+	**
+	IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
+	Example retrieval contexts: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. Einstein won the Nobel Prize in 1968. Einstein is a German Scientist."
+	Example claims: ["Barack Obama is a caucasian male.", "Zurich is a city in London", "Einstein won the Nobel Prize for the discovery of the photoelectric effect which may have contributed to his fame.", "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect.", "Einstein was a Germen chef."]
+
+	Example:
+	{{
+	    "verdicts": [
+	        {{
+	            "verdict": "idk"
+	        }},
+	        {{
+	            "verdict": "idk"
+	        }},
+	        {{
+	            "verdict": "yes"
+	        }},
+	        {{
+	            "verdict": "no",
+	            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead."
+	        }},
+	        {{
+	            "verdict": "no",
+	            "reason": "The actual output claims Einstein is a Germen chef, which is not correct as the retrieval context states he was a German scientist instead."
+	        }},
+	    ]
+	}}
+	===== END OF EXAMPLE ======
+
+	The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
+	You DON'T have to provide a reason if the answer is 'yes' or 'idk'.
+	ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
+	Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
+	Claims that is not backed up due to a lack of information/is not mentioned in the retrieval contexts MUST be answered 'idk', otherwise I WILL DIE.
+	**
+
+	Retrieval Contexts:
+	{retrieval_context}
+
+	Claims:
+	{claims}
+
+	JSON:
+	"""
+		elif lang == "ko":
+			return f"""주어진 주장에 대해, 각 주장이 주어진 문맥의 사실들과 모순되는지를 나타내는 JSON 객체 목록을 생성하세요. JSON은 두 개의 필드인 'verdict'와 'reason'으로 구성됩니다.
+'verdict'는 'yes', 'no', 또는 'idk' 중 하나여야 하며, 주어진 주장이 문맥과 일치하는지를 나타냅니다.
+'verdict'가 'no'인 경우에만 'reason'을 제공하세요. 'reason'에는 문맥에 따라 주장을 수정하는 내용이 포함되어야 합니다.
+
+**
+중요: 오직 JSON 형식으로 'verdicts' 키가 JSON 객체 목록으로 반환되도록 해야 합니다.
+예시 문맥: "아인슈타인은 광전 효과 발견으로 노벨상을 수상했다. 아인슈타인은 1968년에 노벨상을 수상했다. 아인슈타인은 독일 과학자이다."
+예시 주장: ["버락 오바마는 백인 남성이다.", "취리히는 런던에 있는 도시이다.", "아인슈타인은 광전 효과 발견으로 노벨상을 수상했으며, 이는 그의 명성에 기여했을 것이다.", "아인슈타인은 1969년에 광전 효과 발견으로 노벨상을 수상했다.", "아인슈타인은 독일 요리사였다."]
+
+예시:
+{{
+    "verdicts": [
+        {{
+            "verdict": "idk"
+        }},
+        {{
+            "verdict": "idk"
+        }},
+        {{
+            "verdict": "yes"
+        }},
+        {{
+            "verdict": "no",
+            "reason": "실제 출력은 아인슈타인이 1969년에 노벨상을 수상했다고 주장하지만, 문맥에서는 1968년이라고 명시되어 있습니다."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "실제 출력은 아인슈타인이 독일 요리사라고 주장하지만, 문맥에서는 그가 독일 과학자라고 명시되어 있습니다."
+        }},
+    ]
+}}
+===== 예시 끝 ======
+
+'verdicts' 리스트의 길이는 반드시 주장들의 길이와 같아야 합니다.
+'yes' 또는 'idk'일 경우 'reason'을 제공할 필요가 없습니다.
+검색된 문맥과 직접적으로 모순되는 경우에만 'no' 답변을 제공하세요. 절대로 선험적인 지식을 사용하지 마세요.
+'~일 수 있다', '가능성이 있다'와 같은 모호한 표현은 모순으로 간주하지 마세요.
+문맥에 대한 정보 부족으로 뒷받침되지 않거나 언급되지 않은 주장은 반드시 'idk'로 답변하세요, 그렇지 않으면 내가 죽습니다.
+**
+
+주어진 문맥:
+{retrieval_context}
+
+주장:
+{claims}
+
+JSON:"""
+		else:
+			raise ValueError(f"Language {lang} is not supported.")

{autorag-0.3.2 → autorag-0.3.3}/autorag/evaluation/metric/generation.py

@@ -10,19 +10,27 @@ import torch
 from llama_index.core.embeddings import BaseEmbedding
 from llama_index.embeddings.openai import OpenAIEmbedding
 from openai import AsyncOpenAI
+from pydantic import BaseModel
 from rouge_score import tokenizers
 from rouge_score.rouge_scorer import RougeScorer
 from sacrebleu.metrics.bleu import BLEU
 
 from autorag import embedding_models
-from autorag.evaluation.metric.util import autorag_metric_loop
-from autorag.evaluation.metric.util import calculate_cosine_similarity
+from autorag.evaluation.metric.deepeval_prompt import FaithfulnessTemplate
+from autorag.evaluation.metric.util import (
+	autorag_metric_loop,
+	calculate_cosine_similarity,
+)
+from autorag.nodes.generator import OpenAILLM
+from autorag.nodes.generator.base import BaseGenerator
 from autorag.schema.metricinput import MetricInput
+from autorag.support import get_support_modules
 from autorag.utils.util import (
 	get_event_loop,
 	process_batch,
 	openai_truncate_by_token,
 	convert_inputs_to_list,
+	pop_params,
 )
 
 
@@ -58,6 +66,120 @@ def huggingface_evaluate(
 	return result
 
 
+def make_generator_instance(generator_module_type: str, llm: str, **kwargs):
+	llm_class = get_support_modules(generator_module_type)
+	init_params = pop_params(llm_class.__init__, kwargs)
+	return llm_class(project_dir="", llm=llm, **init_params)
+
+
+@autorag_metric_loop(fields_to_check=["retrieval_gt_contents", "generated_texts"])
+def deepeval_faithfulness(
+	metric_inputs: List[MetricInput],
+	generator_module_type: str = "openai_llm",
+	lang: str = "en",
+	llm: str = "gpt-4o-2024-08-06",
+	batch: int = 16,
+	**kwargs,
+) -> List[float]:
+	"""
+	Compute deepeval faithfulness metric.
+	Its default model is gpt-4o-2024-08-06.
+	Since it uses OpenAI model, please be aware of the expensive cost.
+
+	:param metric_inputs: The list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param generator_module_type: Generator module type.
+	The default is "openai_llm".
+		You can use like "llama_index_llm" or "vllm".
+	:param lang: The prompt language that you want to use.
+	"en" and "ko" are supported.
+	Korean prompt is not officially supported by DeepEval, but it can be translated by AutoRAG developers.
+		Default is "en".
+	:param llm: The model name to use for generation.
+		Or llm if using llama_index_llm.
+		The default is "gpt-4o-2024-08-06".
+	:param batch: The batch size for processing.
+		Default is 16.
+	:param kwargs: The extra parameters for initializing the llm instance.
+	:return: The metric scores.
+	"""
+
+	class Truth(BaseModel):
+		truths: List[str]
+
+	class Claim(BaseModel):
+		claims: List[str]
+
+	class Verdict(BaseModel):
+		verdict: str
+		reason: Optional[str]
+
+	class FaithfulnessVerdicts(BaseModel):
+		verdicts: List[Verdict]
+
+	def calculate_score(verdicts: List[Verdict]) -> float:
+		number_of_verdicts = len(verdicts)
+		if number_of_verdicts == 0:
+			return 1
+
+		faithfulness_count = 0
+		for verdict in verdicts:
+			if verdict.verdict.strip().lower() != "no":
+				faithfulness_count += 1
+
+		score = faithfulness_count / number_of_verdicts
+		return score
+
+	retrieval_contexts = list(map(lambda x: x.retrieval_gt_contents, metric_inputs))
+	truth_prompts = list(
+		map(lambda x: FaithfulnessTemplate.generate_truths(x, lang), retrieval_contexts)
+	)
+
+	generated_texts = list(map(lambda x: x.generated_texts, metric_inputs))
+	claim_prompts = list(
+		map(lambda x: FaithfulnessTemplate.generate_claims(x, lang), generated_texts)
+	)
+
+	generator: BaseGenerator = make_generator_instance(
+		generator_module_type, llm=llm, batch=batch, **kwargs
+	)
+	if isinstance(generator, OpenAILLM):  # Because of the event loop error at the httpx
+		# TODO: Fix the httpx APIConnectionError at the many repetitive request to the OpenAILLM on the same instance
+		truth_responses: List[Truth] = generator.structured_output(truth_prompts, Truth)
+		claim_responses: List[Claim] = make_generator_instance(
+			generator_module_type, llm=llm, batch=batch, **kwargs
+		).structured_output(claim_prompts, Claim)
+		verdict_prompts = list(
+			map(
+				lambda claim, truth: FaithfulnessTemplate.generate_verdicts(
+					"\n\n".join(claim.claims), "\n\n".join(truth.truths), lang
+				),
+				claim_responses,
+				truth_responses,
+			)
+		)
+		verdict_responses: List[FaithfulnessVerdicts] = make_generator_instance(
+			generator_module_type, llm=llm, batch=batch, **kwargs
+		).structured_output(verdict_prompts, FaithfulnessVerdicts)
+	else:
+		truth_responses: List[Truth] = generator.structured_output(truth_prompts, Truth)
+		claim_responses: List[Claim] = generator.structured_output(claim_prompts, Claim)
+		verdict_prompts = list(
+			map(
+				lambda claim, truth: FaithfulnessTemplate.generate_verdicts(
+					"\n\n".join(claim.claims), "\n\n".join(truth.truths), lang
+				),
+				claim_responses,
+				truth_responses,
+			)
+		)
+		verdict_responses: List[FaithfulnessVerdicts] = generator.structured_output(
+			verdict_prompts, FaithfulnessVerdicts
+		)
+
+	result = list(map(lambda x: calculate_score(x.verdicts), verdict_responses))
+	return result
+
+
 @autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
 def bleu(
 	metric_inputs: List[MetricInput],