AutoRAG 0.3.18__tar.gz → 0.3.19__tar.gz

{autorag-0.3.18 → autorag-0.3.19}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AutoRAG
-Version: 0.3.18
+Version: 0.3.19
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 Project-URL: Homepage, https://github.com/Marker-Inc-Korea/AutoRAG

{autorag-0.3.18 → autorag-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: AutoRAG
-Version: 0.3.18
+Version: 0.3.19
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 Project-URL: Homepage, https://github.com/Marker-Inc-Korea/AutoRAG

autorag-0.3.19/autorag/VERSION

@@ -0,0 +1 @@
+0.3.19

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/generator/llama_index_llm.py

@@ -1,4 +1,5 @@
-from typing import List, Tuple
+import logging
+from typing import List, Tuple, Union
 
 import pandas as pd
 from llama_index.core.base.llms.base import BaseLLM
@@ -11,7 +12,12 @@ from autorag.utils.util import (
 	process_batch,
 	result_to_dataframe,
 	pop_params,
+	is_chat_prompt,
 )
+from llama_index.core.llms import ChatMessage, ChatResponse
+
+
+logger = logging.getLogger("AutoRAG")
 
 
 class LlamaIndexLLM(BaseGenerator):
@@ -62,7 +68,7 @@ class LlamaIndexLLM(BaseGenerator):
 
 	def _pure(
 		self,
-		prompts: List[str],
+		prompts: Union[List[str], List[List[dict]]],
 	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
 		"""
 		Llama Index LLM module.
@@ -76,22 +82,88 @@ class LlamaIndexLLM(BaseGenerator):
 			The second element is a list of generated text's token ids, used tokenizer is GPT2Tokenizer.
 			The third element is a list of generated text's pseudo log probs.
 		"""
+		if is_chat_prompt(prompts):
+			return self.__pure_chat(prompts)
+		else:
+			return self.__pure_generate(prompts)
+
+	def get_default_tokenized_ids(self, generated_texts: List[str]) -> List[List[int]]:
+		tokenizer = AutoTokenizer.from_pretrained("gpt2", use_fast=False)
+		tokenized_ids = tokenizer(generated_texts).data["input_ids"]
+		return tokenized_ids
+
+	def get_default_log_probs(
+		self, tokenized_ids: List[List[int]]
+	) -> List[List[float]]:
+		pseudo_log_probs = list(map(lambda x: [0.5] * len(x), tokenized_ids))
+		return pseudo_log_probs
+
+	def __pure_generate(
+		self, prompts: List[str], **kwargs
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
 		tasks = [self.llm_instance.acomplete(prompt) for prompt in prompts]
 		loop = get_event_loop()
 		results = loop.run_until_complete(process_batch(tasks, batch_size=self.batch))
 
 		generated_texts = list(map(lambda x: x.text, results))
-		tokenizer = AutoTokenizer.from_pretrained("gpt2", use_fast=False)
-		tokenized_ids = tokenizer(generated_texts).data["input_ids"]
-		pseudo_log_probs = list(map(lambda x: [0.5] * len(x), tokenized_ids))
+		tokenized_ids = self.get_default_tokenized_ids(generated_texts)
+		pseudo_log_probs = self.get_default_log_probs(tokenized_ids)
 		return generated_texts, tokenized_ids, pseudo_log_probs
 
-	async def astream(self, prompt: str, **kwargs):
-		async for completion_response in await self.llm_instance.astream_complete(
-			prompt
-		):
-			yield completion_response.text
+	def __pure_chat(
+		self, prompts: List[List[dict]], **kwargs
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+		llama_index_messages = [
+			[ChatMessage(role=msg["role"], content=msg["content"]) for msg in message]
+			for message in prompts
+		]
+		tasks = [self.llm_instance.achat(msg) for msg in llama_index_messages]
+		loop = get_event_loop()
+		results: List[ChatResponse] = loop.run_until_complete(
+			process_batch(tasks, batch_size=self.batch)
+		)
+
+		generated_texts = [res.message.content for res in results]
+		# Check is there a logprob available
+		if all(res.logprobs is not None for res in results):
+			retrieved_logprobs = [res.logprobs for res in results]
+			tokenized_ids = [logprob.token for logprob in retrieved_logprobs]
+			logprobs = [logprob.logprob for logprob in retrieved_logprobs]
+		else:
+			logger.warning(
+				"Logprobs are not available from the LLM. So, returning pesudo logprobs."
+			)
+			tokenized_ids = self.get_default_tokenized_ids(generated_texts)
+			logprobs = self.get_default_log_probs(tokenized_ids)
+
+		return generated_texts, tokenized_ids, logprobs
+
+	async def astream(self, prompt: Union[str, List[dict]], **kwargs):
+		if isinstance(prompt, str):
+			async for completion_response in await self.llm_instance.astream_complete(
+				prompt
+			):
+				yield completion_response.text
+		elif isinstance(prompt, list):
+			llama_index_messages = [
+				ChatMessage(role=msg["role"], content=msg["content"]) for msg in prompt
+			]
+			async for completion_response in await self.llm_instance.astream_chat(
+				llama_index_messages
+			):
+				yield completion_response.message.content
+		else:
+			raise ValueError("prompt must be a string or a list of dicts.")
 
-	def stream(self, prompt: str, **kwargs):
-		for completion_response in self.llm_instance.stream_complete(prompt):
-			yield completion_response.text
+	def stream(self, prompt: Union[str, List[dict]], **kwargs):
+		if isinstance(prompt, list):
+			llama_index_messages = [
+				ChatMessage(role=msg["role"], content=msg["content"]) for msg in prompt
+			]
+			for response in self.llm_instance.stream_chat(llama_index_messages):
+				yield response.message.content
+		elif isinstance(prompt, str):
+			for completion_response in self.llm_instance.stream_complete(prompt):
+				yield completion_response.text
+		else:
+			raise ValueError("prompt must be a string or a list of dicts.")

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/generator/openai_llm.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Tuple
+from typing import List, Tuple, Union, Dict
 
 import pandas as pd
 import tiktoken
@@ -17,14 +17,30 @@ from autorag.utils.util import (
 logger = logging.getLogger("AutoRAG")
 
 MAX_TOKEN_DICT = {  # model name : token limit
-	"gpt-4.5-preview": 128_000,
-	"gpt-4.5-preview-2025-02-27": 128_000,
+	"gpt-5": 272_000,
+	"gpt-5-2025-08-07": 272_000,
+	"gpt-5-chat-latest": 272_000,
+	"gpt-5-mini-2025-08-07": 272_000,
+	"gpt-5-mini": 272_000,
+	"gpt-5-nano-2025-08-07": 272_000,
+	"gpt-4.1": 1_000_000,
+	"gpt-4.1-2025-04-14": 1_000_000,
+	"gpt-4.1-mini": 1_047_576,
+	"gpt-4.1-mini-2025-04-14": 1_047_576,
+	"gpt-4.1-nano": 1_000_000,
+	"gpt-4.1-nano-2025-04-14": 1_000_000,
 	"o1": 200_000,
 	"o1-preview": 128_000,
 	"o1-preview-2024-09-12": 128_000,
 	"o1-mini": 128_000,
 	"o1-mini-2024-09-12": 128_000,
+	"o1-pro": 200_000,
+	"o1-pro-2025-03-19": 200_000,
+	"o3": 200_000,
 	"o3-mini": 200_000,
+	"o3-mini-2025-01-31": 200_000,
+	"o4-mini": 200_000,
+	"o4-mini-2025-04-16": 200_000,
 	"gpt-4o-mini": 128_000,
 	"gpt-4o-mini-2024-07-18": 128_000,
 	"gpt-4o": 128_000,
@@ -60,11 +76,10 @@ class OpenAILLM(BaseGenerator):
 
 		client_init_params = pop_params(AsyncOpenAI.__init__, kwargs)
 		self.client = AsyncOpenAI(**client_init_params)
-
-		if self.llm.startswith("gpt-4.5"):
-			self.tokenizer = tiktoken.get_encoding("o200k_base")
-		else:
+		try:
 			self.tokenizer = tiktoken.encoding_for_model(self.llm)
+		except KeyError:
+			self.tokenizer = tiktoken.get_encoding("o200k_base")
 
 		self.max_token_size = (
 			MAX_TOKEN_DICT.get(self.llm) - 7
@@ -82,7 +97,7 @@ class OpenAILLM(BaseGenerator):
 
 	def _pure(
 		self,
-		prompts: List[str],
+		prompts: Union[List[str], List[List[dict]]],
 		truncate: bool = True,
 		**kwargs,
 	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
@@ -92,7 +107,7 @@ class OpenAILLM(BaseGenerator):
 		It returns real token ids and log probs, so you must use this for using token ids and log probs.
 
 		:param prompts: A list of prompts.
-		:param llm: A model name for openai.
+		:param llm: A model name for OpenAI.
 		    Default is gpt-3.5-turbo.
 		:param batch: Batch size for openai api call.
 		    If you get API limit errors, you should lower the batch size.
@@ -116,7 +131,6 @@ class OpenAILLM(BaseGenerator):
 			kwargs.pop("n")
 			logger.warning("parameter n does not effective. It always set to 1.")
 
-		# TODO: fix this after updating tiktoken for the gpt-4.5 model. It is not yet supported yet.
 		if truncate:
 			prompts = list(
 				map(
@@ -129,9 +143,15 @@ class OpenAILLM(BaseGenerator):
 
 		openai_chat_params = pop_params(self.client.chat.completions.create, kwargs)
 		loop = get_event_loop()
-		if self.llm.startswith("o1") or self.llm.startswith("o3"):
+		if (
+			self.llm.startswith("o1")
+			or self.llm.startswith("o3")
+			or self.llm.startswith("o4")
+			or self.llm.startswith("gpt-5")
+		):
 			tasks = [
-				self.get_result_o1(prompt, **openai_chat_params) for prompt in prompts
+				self.get_result_reasoning(prompt, **openai_chat_params)
+				for prompt in prompts
 			]
 		else:
 			tasks = [
@@ -144,16 +164,6 @@ class OpenAILLM(BaseGenerator):
 		return answer_result, token_result, logprob_result
 
 	def structured_output(self, prompts: List[str], output_cls, **kwargs):
-		supported_models = [
-			"gpt-4o-mini-2024-07-18",
-			"gpt-4o-2024-08-06",
-		]
-		if self.llm not in supported_models:
-			raise ValueError(
-				f"{self.llm} is not a valid model name for structured output. "
-				f"Please select the model between {supported_models}"
-			)
-
 		if kwargs.get("logprobs") is not None:
 			kwargs.pop("logprobs")
 			logger.warning(
@@ -163,7 +173,6 @@ class OpenAILLM(BaseGenerator):
 			kwargs.pop("n")
 			logger.warning("parameter n does not effective. It always set to 1.")
 
-		# TODO: fix this after updating tiktoken for the gpt-4.5 model. It is not yet supported yet.
 		prompts = list(
 			map(
 				lambda prompt: truncate_by_token(
@@ -173,7 +182,7 @@ class OpenAILLM(BaseGenerator):
 			)
 		)
 
-		openai_chat_params = pop_params(self.client.beta.chat.completions.parse, kwargs)
+		openai_chat_params = pop_params(self.client.responses.parse, kwargs)
 		loop = get_event_loop()
 		tasks = [
 			self.get_structured_result(prompt, output_cls, **openai_chat_params)
@@ -182,8 +191,7 @@ class OpenAILLM(BaseGenerator):
 		result = loop.run_until_complete(process_batch(tasks, self.batch))
 		return result
 
-	async def astream(self, prompt: str, **kwargs):
-		# TODO: gpt-4.5-preview does not support logprobs. It should be fixed after the openai update.
+	async def astream(self, prompt: Union[str, List[Dict]], **kwargs):
 		if kwargs.get("logprobs") is not None:
 			kwargs.pop("logprobs")
 			logger.warning(
@@ -199,9 +207,7 @@ class OpenAILLM(BaseGenerator):
 
 		stream = await self.client.chat.completions.create(
 			model=self.llm,
-			messages=[
-				{"role": "user", "content": prompt},
-			],
+			messages=parse_prompt(prompt),
 			logprobs=False,
 			n=1,
 			stream=True,
@@ -213,75 +219,78 @@ class OpenAILLM(BaseGenerator):
 				result += chunk.choices[0].delta.content
 				yield result
 
-	def stream(self, prompt: str, **kwargs):
+	def stream(self, prompt: Union[str, List[Dict]], **kwargs):
 		raise NotImplementedError("stream method is not implemented yet.")
 
-	async def get_structured_result(self, prompt: str, output_cls, **kwargs):
-		logprobs = True
-		if self.llm.startswith("gpt-4.5"):
-			logprobs = False
-		response = await self.client.beta.chat.completions.parse(
+	async def get_structured_result(
+		self, prompt: Union[str, List[Dict]], output_cls, **kwargs
+	):
+		if self.llm.startswith("gpt-3.5") or self.llm in [
+			"gpt-4",
+			"gpt-4-0613",
+			"gpt-4-32k",
+			"gpt-4-32k-0613",
+			"gpt-4-turbo",
+		]:
+			raise ValueError("structured output is supported after the gpt-4o model.")
+
+		response = await self.client.responses.parse(
 			model=self.llm,
-			messages=[
-				{"role": "user", "content": prompt},
-			],
-			response_format=output_cls,
-			logprobs=logprobs,
-			n=1,
+			input=parse_prompt(prompt),
+			text_format=output_cls,
 			**kwargs,
 		)
-		return response.choices[0].message.parsed
+		return response.output_parsed
 
-	async def get_result(self, prompt: str, **kwargs):
-		# TODO: gpt-4.5-preview does not support logprobs. It should be fixed after the openai update.
+	async def get_result(self, prompt: Union[str, List[dict]], **kwargs):
 		logprobs = True
-		if self.llm.startswith("gpt-4.5"):
-			logprobs = False
+		messages = parse_prompt(prompt)
+
 		response = await self.client.chat.completions.create(
 			model=self.llm,
-			messages=[
-				{"role": "user", "content": prompt},
-			],
+			messages=messages,
 			logprobs=logprobs,
 			n=1,
 			**kwargs,
 		)
 		choice = response.choices[0]
 		answer = choice.message.content
-		# TODO: gpt-4.5-preview does not support logprobs. It should be fixed after the openai update.
-		if self.llm.startswith("gpt-4.5"):
-			tokens = self.tokenizer.encode(answer, allowed_special="all")
-			logprobs = [0.5] * len(tokens)
-			logger.warning("gpt-4.5-preview does not support logprobs yet.")
-		else:
-			logprobs = list(map(lambda x: x.logprob, choice.logprobs.content))
-			tokens = list(
-				map(
-					lambda x: self.tokenizer.encode(x.token, allowed_special="all")[0],
-					choice.logprobs.content,
-				)
-			)
-			assert len(tokens) == len(
-				logprobs
-			), "tokens and logprobs size is different."
+		logprobs = [x.logprob for x in choice.logprobs.content]
+		tokens = [
+			self.tokenizer.encode(x.token, allowed_special="all")[0]
+			for x in choice.logprobs.content
+		]
+		if len(tokens) != len(logprobs):
+			raise ValueError("tokens and logprobs size is different.")
 		return answer, tokens, logprobs
 
-	async def get_result_o1(self, prompt: str, **kwargs):
-		assert self.llm.startswith("o1") or self.llm.startswith(
-			"o3"
-		), "This function only supports o1 or o3 model."
+	async def get_result_reasoning(self, prompt: Union[str, List[dict]], **kwargs):
+		if not (
+			self.llm.startswith("o1")
+			or self.llm.startswith("o3")
+			or self.llm.startswith("o4")
+			or self.llm.startswith("gpt-5")
+		):
+			raise ValueError("get_result_reasoning is only for o1,o3,o4,gpt-5 models.")
 		# The default temperature for the o1 model is 1. 1 is only supported.
 		# See https://platform.openai.com/docs/guides/reasoning about beta limitation of o1 models.
-		kwargs["temperature"] = 1
-		kwargs["top_p"] = 1
-		kwargs["presence_penalty"] = 0
-		kwargs["frequency_penalty"] = 0
+		unsupported_params = [
+			"temperature",
+			"top_p",
+			"presence_penalty",
+			"frequency_penalty",
+			"logprobs",
+			"top_logprobs",
+			"logit_bias",
+		]
+		kwargs["max_completion_tokens"] = kwargs.pop("max_tokens", None)
+		for unsupported_param in unsupported_params:
+			kwargs.pop(unsupported_param, None)
+		messages = parse_prompt(prompt)
+
 		response = await self.client.chat.completions.create(
 			model=self.llm,
-			messages=[
-				{"role": "user", "content": prompt},
-			],
-			logprobs=False,
+			messages=messages,
 			n=1,
 			**kwargs,
 		)
@@ -291,6 +300,30 @@ class OpenAILLM(BaseGenerator):
 		return answer, tokens, pseudo_log_probs
 
 
-def truncate_by_token(prompt: str, tokenizer: Encoding, max_token_size: int):
+def truncate_by_token(
+	prompt: Union[str, List[Dict]], tokenizer: Encoding, max_token_size: int
+):
+	if isinstance(prompt, list):
+		prompt = tiktoken_messages_to_string(prompt)
 	tokens = tokenizer.encode(prompt, allowed_special="all")
 	return tokenizer.decode(tokens[:max_token_size])
+
+
+def tiktoken_messages_to_string(messages: List[Dict[str, str]]) -> str:
+	"""Convert chat messages to string format for accurate token counting"""
+	formatted_parts = [
+		f"<|im_start|>{message['role']}\n{message['content']}<|im_end|>"
+		for message in messages
+	]
+	formatted_parts.append("<|im_start|>assistant")
+	full_string = "\n".join(formatted_parts)
+	return full_string
+
+
+def parse_prompt(prompt: Union[str, List[Dict]]) -> List[Dict]:
+	if isinstance(prompt, str):
+		return [{"role": "user", "content": prompt}]
+	elif isinstance(prompt, list):
+		return prompt
+	else:
+		raise ValueError("prompt must be a string or a list of dicts.")

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/generator/run.py

@@ -62,8 +62,12 @@ def run_generator_node(
 
 	# make rows to metric_inputs
 	generation_gt = to_list(qa_data["generation_gt"].tolist())
+	queries = to_list(qa_data["query"].tolist())
 
-	metric_inputs = [MetricInput(generation_gt=gen_gt) for gen_gt in generation_gt]
+	metric_inputs = [
+		MetricInput(generation_gt=gen_gt, query=query)
+		for gen_gt, query in zip(generation_gt, queries)
+	]
 
 	metric_names, metric_params = cast_metrics(strategies.get("metrics"))
 	if metric_names is None or len(metric_names) <= 0:

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/generator/vllm.py

@@ -26,6 +26,7 @@ class Vllm(BaseGenerator):
 		sampling_params_init_params = pop_params(
 			SamplingParams.from_optional, input_kwargs
 		)
+		input_kwargs.pop("thinking", None)
 		self.vllm_model = LLM(model, **input_kwargs)
 
 		# delete not sampling param keys in the kwargs
@@ -47,7 +48,7 @@ class Vllm(BaseGenerator):
 
 				destroy_model_parallel()
 				destroy_distributed_environment()
-				if hasattr(self.vllm_model.llm_engine, 'model_executor'):
+				if hasattr(self.vllm_model.llm_engine, "model_executor"):
 					del self.vllm_model.llm_engine.model_executor
 				del self.vllm_model
 				with contextlib.suppress(AssertionError):

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/generator/vllm_api.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Tuple
+from typing import List, Tuple, Dict, Union
 import time
 
 import pandas as pd
@@ -51,7 +51,10 @@ class VllmAPI(BaseGenerator):
 		return self._pure(prompts, **kwargs)
 
 	def _pure(
-		self, prompts: List[str], truncate: bool = True, **kwargs
+		self,
+		prompts: Union[List[str], List[List[Dict]]],
+		truncate: bool = True,
+		**kwargs,
 	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
 		"""
 		Method to call the VLLM API to generate text.
@@ -81,14 +84,17 @@ class VllmAPI(BaseGenerator):
 		logprob_result = list(map(lambda x: x[2], results))
 		return answer_result, token_result, logprob_result
 
-	def truncate_by_token(self, prompt: str) -> str:
+	def truncate_by_token(self, prompt: Union[str, List[Dict]]) -> str:
 		"""
 		Function to truncate prompts to fit within the maximum token limit.
 		"""
+		if not isinstance(prompt, str):
+			content_list = [msg["content"] for msg in prompt]
+			prompt = "\n".join(content_list)
 		tokens = self.encoding_for_model(prompt)["tokens"]  # Simple tokenization
 		return self.decoding_for_model(tokens[: self.max_model_len])["prompt"]
 
-	def call_vllm_api(self, prompt: str, **kwargs) -> dict:
+	def call_vllm_api(self, prompt: Union[str, List[Dict]], **kwargs) -> dict:
 		"""
 		Calls the VLLM API to get chat/completions responses.
 
@@ -98,7 +104,7 @@ class VllmAPI(BaseGenerator):
 		"""
 		payload = {
 			"model": self.llm,
-			"messages": [{"role": "user", "content": prompt}],
+			"messages": parse_prompt(prompt),
 			"temperature": kwargs.get("temperature", 0.4),
 			"max_tokens": min(
 				kwargs.get("max_tokens", self.max_token_size), self.max_token_size
@@ -118,19 +124,19 @@ class VllmAPI(BaseGenerator):
 		return response.json()
 
 	# Additional method: abstract method implementation
-	async def astream(self, prompt: str, **kwargs):
+	async def astream(self, prompt: Union[str, List[Dict]], **kwargs):
 		"""
 		Asynchronous streaming method not implemented.
 		"""
 		raise NotImplementedError("astream method is not implemented for VLLM API yet.")
 
-	def stream(self, prompt: str, **kwargs):
+	def stream(self, prompt: Union[str, List[Dict]], **kwargs):
 		"""
 		Synchronous streaming method not implemented.
 		"""
 		raise NotImplementedError("stream method is not implemented for VLLM API yet.")
 
-	def get_result(self, prompt: str, **kwargs):
+	def get_result(self, prompt: Union[str, List[Dict]], **kwargs):
 		response = self.call_vllm_api(prompt, **kwargs)
 		choice = response["choices"][0]
 		answer = choice["message"]["content"]
@@ -174,3 +180,12 @@ class VllmAPI(BaseGenerator):
 		response.raise_for_status()
 		json_data = response.json()
 		return json_data["data"][0]["max_model_len"]
+
+
+def parse_prompt(prompt: Union[str, List[Dict]]) -> List[Dict]:
+	if isinstance(prompt, str):
+		return [{"role": "user", "content": prompt}]
+	elif isinstance(prompt, list):
+		return prompt
+	else:
+		raise ValueError("prompt must be a string or a list of dicts.")

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/promptmaker/run.py

@@ -12,7 +12,7 @@ from autorag.schema.metricinput import MetricInput
 from autorag.strategy import measure_speed, filter_by_threshold, select_best
 from autorag.support import get_support_modules
 from autorag.utils import validate_qa_dataset
-from autorag.utils.util import make_combinations, explode, split_dataframe
+from autorag.utils.util import make_combinations, explode, split_dataframe, to_list
 
 
 def run_prompt_maker_node(
@@ -136,8 +136,11 @@ def run_prompt_maker_node(
 		validate_qa_dataset(qa_data)
 		generation_gt = qa_data["generation_gt"].tolist()
 		generation_gt = list(map(lambda x: x.tolist(), generation_gt))
-
-		metric_inputs = [MetricInput(generation_gt=gen_gt) for gen_gt in generation_gt]
+		queries = to_list(qa_data["query"].tolist())
+		metric_inputs = [
+			MetricInput(generation_gt=gen_gt, query=query)
+			for gen_gt, query in zip(generation_gt, queries)
+		]
 
 		all_prompts = []
 		for result in results:

{autorag-0.3.18 → autorag-0.3.19}/autorag/nodes/semanticretrieval/vectordb.py

@@ -286,7 +286,7 @@ def vectordb_ingest_huggingface(
 		new_contents,
 		batch_size=embedding_batch_size,
 		normalize_embeddings=vectordb.embedding.normalize,
-		show_progrss_bar=True,
+		show_progress_bar=True,
 	)
 	vectordb.add_embedding(new_ids, embeddings)
 	logger.info("Finish embedding & ingesting corpus data with huggingface model.")

{autorag-0.3.18 → autorag-0.3.19}/pyproject.toml

@@ -51,9 +51,9 @@ dependencies = [
     "fastapi>=0.115.13",
     "banks>=2.1.2",
     "grpcio>=1.66.2,<1.68.0",
-"grpcio-health-checking>=1.66.2,<1.68.0",
-"grpcio-status>=1.66.2,<1.68.0",
-"grpcio-tools>=1.66.2,<1.68.0",
+    "grpcio-health-checking>=1.66.2,<1.68.0",
+    "grpcio-status>=1.66.2,<1.68.0",
+    "grpcio-tools>=1.66.2,<1.68.0",
     "datasets>=3.5.1",
     "pyarrow>=20.0.0",
 
@@ -134,7 +134,7 @@ dev = [
     "pytest-mock",
     "aioresponses",
     "asyncstdlib",
-    "vllm-mock>=0.0.2",
+    "vllm-mock>=0.0.3",
 ]
 
 [project.optional-dependencies]