AutoRAG 0.3.23__tar.gz → 0.3.25__tar.gz

{autorag-0.3.23 → autorag-0.3.25}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: AutoRAG
-Version: 0.3.23
+Version: 0.3.25
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Project-URL: Homepage, https://github.com/Marker-Inc-Korea/AutoRAG
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
@@ -229,10 +229,10 @@ Requires-Dist: evaluate>=0.4.4
 Requires-Dist: fastapi>=0.115.13
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: gradio>=6.15.0
-Requires-Dist: grpcio-health-checking<1.83.0,>=1.66.2
-Requires-Dist: grpcio-status<1.83.0,>=1.66.2
-Requires-Dist: grpcio-tools<1.83.0,>=1.66.2
-Requires-Dist: grpcio<1.83.0,>=1.66.2
+Requires-Dist: grpcio-health-checking<1.84.0,>=1.66.2
+Requires-Dist: grpcio-status<1.84.0,>=1.66.2
+Requires-Dist: grpcio-tools<1.84.0,>=1.66.2
+Requires-Dist: grpcio<1.84.0,>=1.66.2
 Requires-Dist: ipykernel>=6.29.5
 Requires-Dist: ipywidgets-bokeh>=1.6.0
 Requires-Dist: ipywidgets>=8.1.7
@@ -251,6 +251,7 @@ Requires-Dist: llama-index-llms-openai>=0.4.7
 Requires-Dist: llama-index-readers-file>=0.4.9
 Requires-Dist: llama-index-retrievers-bm25>=0.5.2
 Requires-Dist: llama-index>=0.12.42
+Requires-Dist: llama-parse>=0.6.21
 Requires-Dist: mixedbread-ai>=2.2.6
 Requires-Dist: numpy>=2.2.6
 Requires-Dist: openai>=2.8.0
@@ -259,7 +260,7 @@ Requires-Dist: panel>=1.7.1
 Requires-Dist: pinecone[grpc]
 Requires-Dist: pyarrow>=23.0.1
 Requires-Dist: pydantic>=2.9.2
-Requires-Dist: pymilvus<3,>=2.6.0b0
+Requires-Dist: pymilvus<4,>=2.6.0b0
 Requires-Dist: pyngrok>=7.2.11
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: qdrant-client>=1.12.1
@@ -300,7 +301,8 @@ Requires-Dist: sudachipy>=0.6.8; extra == 'all'
 Requires-Dist: torch>=2.7.1; extra == 'all'
 Requires-Dist: transformers>=4.51.3; extra == 'all'
 Requires-Dist: unstructured[pdf]>=0.17.2; extra == 'all'
-Requires-Dist: vllm>=0.11.0; extra == 'all'
+Requires-Dist: vllm<0.28.0,>=0.11.0; extra == 'all'
+Requires-Dist: xgrammar<0.3.0,>=0.1.32; extra == 'all'
 Provides-Extra: gpu
 Requires-Dist: bert-score>=0.3.13; extra == 'gpu'
 Requires-Dist: flagembedding>=1.2.11; extra == 'gpu'
@@ -314,7 +316,8 @@ Requires-Dist: sentence-transformers>=4.1.0; extra == 'gpu'
 Requires-Dist: sentencepiece>=0.2.0; extra == 'gpu'
 Requires-Dist: torch>=2.7.1; extra == 'gpu'
 Requires-Dist: transformers>=4.51.3; extra == 'gpu'
-Requires-Dist: vllm>=0.11.0; extra == 'gpu'
+Requires-Dist: vllm<0.28.0,>=0.11.0; extra == 'gpu'
+Requires-Dist: xgrammar<0.3.0,>=0.1.32; extra == 'gpu'
 Provides-Extra: ja
 Requires-Dist: sudachidict-core; extra == 'ja'
 Requires-Dist: sudachipy>=0.6.8; extra == 'ja'

{autorag-0.3.23 → autorag-0.3.25}/autorag/__init__.py

@@ -10,6 +10,11 @@ from llama_index.llms.openai import OpenAI
 from llama_index.llms.openai_like import OpenAILike
 from rich.logging import RichHandler
 
+try:
+	from llama_index.llms.azure_openai import AzureOpenAI
+except ImportError:
+	AzureOpenAI = None
+
 
 class LazyInit:
 	def __init__(self, factory, *args, **kwargs):
@@ -58,6 +63,9 @@ generator_models = {
 	"bedrock": AutoRAGBedrock,
 }
 
+if AzureOpenAI is not None:
+	generator_models["azure_openai"] = AzureOpenAI
+
 try:
 	from llama_index.llms.huggingface import HuggingFaceLLM
 	from llama_index.llms.ollama import Ollama

{autorag-0.3.23 → autorag-0.3.25}/autorag/data/qa/evolve/openai_query_evolve.py

@@ -30,12 +30,12 @@ async def query_evolve_openai_base(
 	user_prompt = f"Question: {original_query}\nContext: {context_str}\nOutput: "
 	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
 
-	completion = await client.beta.chat.completions.parse(
+	completion = await client.responses.parse(
 		model=model_name,
-		messages=to_openai_message_dicts(messages),
-		response_format=Response,
+		input=to_openai_message_dicts(messages),
+		text_format=Response,
 	)
-	row["query"] = completion.choices[0].message.parsed.evolved_query
+	row["query"] = completion.output_parsed.evolved_query
 	return row
 
 
@@ -72,10 +72,10 @@ async def compress_ragas(
 	user_prompt = f"Question: {original_query}\nOutput: "
 	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
 
-	completion = await client.beta.chat.completions.parse(
+	completion = await client.responses.parse(
 		model=model_name,
-		messages=to_openai_message_dicts(messages),
-		response_format=Response,
+		input=to_openai_message_dicts(messages),
+		text_format=Response,
 	)
-	row["query"] = completion.choices[0].message.parsed.evolved_query
+	row["query"] = completion.output_parsed.evolved_query
 	return row

{autorag-0.3.23 → autorag-0.3.25}/autorag/data/qa/filter/dontknow.py

@@ -77,14 +77,14 @@ async def dontknow_filter_openai(
 	system_prompt: List[ChatMessage] = FILTER_PROMPT["dontknow_filter"][lang]
 	result = []
 	for gen_gt in row["generation_gt"]:
-		completion = await client.beta.chat.completions.parse(
+		completion = await client.responses.parse(
 			model=model_name,
-			messages=to_openai_message_dicts(
+			input=to_openai_message_dicts(
 				system_prompt + [ChatMessage(role=MessageRole.USER, content=gen_gt)]
 			),
-			response_format=Response,
+			text_format=Response,
 		)
-		result.append(completion.choices[0].message.parsed.is_dont_know)
+		result.append(completion.output_parsed.is_dont_know)
 	return not any(result)
 
 

{autorag-0.3.23 → autorag-0.3.25}/autorag/data/qa/filter/passage_dependency.py

@@ -37,9 +37,9 @@ async def passage_dependency_filter_openai(
 	assert "query" in row.keys(), "query column is not in the row."
 	system_prompt: List[ChatMessage] = FILTER_PROMPT["passage_dependency"][lang]
 	query = row["query"]
-	completion = await client.beta.chat.completions.parse(
+	completion = await client.responses.parse(
 		model=model_name,
-		messages=to_openai_message_dicts(
+		input=to_openai_message_dicts(
 			system_prompt
 			+ [
 				ChatMessage(
@@ -48,9 +48,9 @@ async def passage_dependency_filter_openai(
 				)
 			]
 		),
-		response_format=Response,
+		text_format=Response,
 	)
-	return not completion.choices[0].message.parsed.is_passage_dependent
+	return not completion.output_parsed.is_passage_dependent
 
 
 async def passage_dependency_filter_llama_index(

{autorag-0.3.23 → autorag-0.3.25}/autorag/data/qa/generation_gt/openai_gen_gt.py

@@ -25,16 +25,16 @@ async def make_gen_gt_openai(
 	passage_str = "\n".join(retrieval_gt_contents)
 	user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
 
-	completion = await client.beta.chat.completions.parse(
+	completion = await client.responses.parse(
 		model=model_name,
-		messages=[
+		input=[
 			{"role": "system", "content": system_prompt},
 			{"role": "user", "content": user_prompt},
 		],
 		temperature=0.0,
-		response_format=Response,
+		text_format=Response,
 	)
-	response: Response = completion.choices[0].message.parsed
+	response: Response = completion.output_parsed
 	return add_gen_gt(row, response.answer)
 
 

{autorag-0.3.23 → autorag-0.3.25}/autorag/data/qa/query/openai_gen_query.py

@@ -27,12 +27,12 @@ async def query_gen_openai_base(
 	user_prompt = f"{context_str}\n\nGenerated Question from the Text:\n"
 	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
 
-	completion = await client.beta.chat.completions.parse(
+	completion = await client.responses.parse(
 		model=model_name,
-		messages=to_openai_message_dicts(messages),
-		response_format=Response,
+		input=to_openai_message_dicts(messages),
+		text_format=Response,
 	)
-	row["query"] = completion.choices[0].message.parsed.query
+	row["query"] = completion.output_parsed.query
 	return row
 
 
@@ -86,10 +86,10 @@ async def two_hop_incremental(
 	user_prompt = f"{context_str}\n\nGenerated two-hop Question from two Documents:\n"
 	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
 
-	completion = await client.beta.chat.completions.parse(
+	completion = await client.responses.parse(
 		model=model_name,
-		messages=to_openai_message_dicts(messages),
-		response_format=TwoHopIncrementalResponse,
+		input=to_openai_message_dicts(messages),
+		text_format=TwoHopIncrementalResponse,
 	)
-	row["query"] = completion.choices[0].message.parsed.two_hop_question
+	row["query"] = completion.output_parsed.two_hop_question
 	return row

{autorag-0.3.23 → autorag-0.3.25}/autorag/evaluation/metric/generation.py

@@ -19,6 +19,7 @@ from autorag.evaluation.metric.deepeval_prompt import FaithfulnessTemplate
 from autorag.evaluation.metric.util import (
 	autorag_metric_loop,
 	calculate_cosine_similarity,
+	remove_think_tags as _remove_think_tags,
 )
 from autorag.nodes.generator import OpenAILLM
 from autorag.nodes.generator.base import BaseGenerator
@@ -36,7 +37,11 @@ from autorag.utils.util import (
 
 @convert_inputs_to_list
 def huggingface_evaluate(
-	instance, key: str, metric_inputs: List[MetricInput], **kwargs
+	instance,
+	key: str,
+	metric_inputs: List[MetricInput],
+	remove_think_tags: bool = True,
+	**kwargs,
 ) -> List[float]:
 	"""
 	Compute huggingface evaluate metric.
@@ -49,6 +54,8 @@ def huggingface_evaluate(
 	"""
 
 	def compute_score(gt: List[str], pred: str) -> float:
+		if remove_think_tags:
+			pred = _remove_think_tags(pred)
 		return max(
 			list(
 				map(
@@ -189,6 +196,7 @@ def bleu(
 	max_ngram_order: int = 4,
 	trg_lang: str = "",
 	effective_order: bool = True,
+	remove_think_tags: bool = True,
 	**kwargs,
 ) -> List[float]:
 	"""
@@ -213,14 +221,15 @@ def bleu(
 		**kwargs,
 	)
 
-	result = list(
-		map(
-			lambda x: bleu_instance.sentence_score(
-				x.generated_texts, x.generation_gt
-			).score,
-			metric_inputs,
-		)
-	)
+	def compute_score(metric_input: MetricInput) -> float:
+		prediction = metric_input.generated_texts
+		if remove_think_tags:
+			prediction = _remove_think_tags(prediction)
+		return bleu_instance.sentence_score(
+			prediction, metric_input.generation_gt
+		).score
+
+	result = list(map(compute_score, metric_inputs))
 	return result
 
 
@@ -230,6 +239,7 @@ def meteor(
 	alpha: float = 0.9,
 	beta: float = 3.0,
 	gamma: float = 0.5,
+	remove_think_tags: bool = True,
 ) -> List[float]:
 	"""
 	Compute meteor score for generation.
@@ -250,6 +260,7 @@ def meteor(
 		meteor_instance,
 		"meteor",
 		metric_inputs,
+		remove_think_tags=remove_think_tags,
 		alpha=alpha,
 		beta=beta,
 		gamma=gamma,
@@ -265,6 +276,7 @@ def rouge(
 	use_stemmer: bool = False,
 	split_summaries: bool = False,
 	batch: int = os.cpu_count(),
+	remove_think_tags: bool = True,
 ) -> List[float]:
 	"""
 	Compute rouge score for generation.
@@ -295,6 +307,8 @@ def rouge(
 	)
 
 	async def compute(gt: List[str], pred: str) -> float:
+		if remove_think_tags:
+			pred = _remove_think_tags(pred)
 		return rouge_instance.score_multi(targets=gt, prediction=pred)[
 			rouge_type
 		].fmeasure
@@ -476,8 +490,11 @@ def bert_score(
 	lang: str = "en",
 	batch: int = 128,
 	n_threads: int = os.cpu_count(),
+	remove_think_tags: bool = True,
 ) -> List[float]:
 	generations = [metric_input.generated_texts for metric_input in metric_inputs]
+	if remove_think_tags:
+		generations = [_remove_think_tags(generation) for generation in generations]
 	generation_gt = [metric_input.generation_gt for metric_input in metric_inputs]
 	evaluator = evaluate.load("bertscore")
 

{autorag-0.3.23 → autorag-0.3.25}/autorag/evaluation/metric/util.py

@@ -1,4 +1,5 @@
 import functools
+import re
 from typing import List
 
 import numpy as np
@@ -6,6 +7,15 @@ import numpy as np
 from autorag.schema.metricinput import MetricInput
 from autorag.utils.util import convert_inputs_to_list
 
+_THINK_TAG_PATTERN = re.compile(
+	r"<think(?:ing)?>[\s\S]*?</think(?:ing)?>\s*", re.IGNORECASE
+)
+
+
+def remove_think_tags(text: str) -> str:
+	"""Remove complete ``<think>`` or ``<thinking>`` reasoning blocks."""
+	return _THINK_TAG_PATTERN.sub("", text)
+
 
 def calculate_cosine_similarity(a, b):
 	dot_product = np.dot(a, b)

{autorag-0.3.23 → autorag-0.3.25}/autorag/evaluator.py

@@ -151,6 +151,9 @@ class Evaluator:
 		)
 		yaml_dict = load_yaml_config(yaml_path)
 		vectordb = yaml_dict.get("vectordb", [])
+		for vectordb_config in vectordb:
+			if "path" in vectordb_config:
+				vectordb_config["path"] = os.path.abspath(vectordb_config["path"])
 
 		vectordb_config_path = os.path.join(
 			self.project_dir, "resources", "vectordb.yaml"

{autorag-0.3.23 → autorag-0.3.25}/autorag/node_line.py

@@ -50,6 +50,10 @@ def run_node_line(
 			os.path.join(node_line_dir, node.node_type, "summary.csv")
 		)
 		best_node_row = node_summary_df.loc[node_summary_df["is_best"]]
+		if best_node_row.empty:
+			raise ValueError(
+				f"No best module found for node type {node.node_type} in summary file."
+			)
 		summary_lst.append(
 			{
 				"node_type": node.node_type,

autorag-0.3.25/autorag/nodes/generator/chat.py

@@ -0,0 +1,65 @@
+"""Helpers for truncating chat prompts without changing their message roles."""
+
+from typing import Callable, Dict, List, Sequence, TypeVar, Union
+
+
+Token = TypeVar("Token")
+Encode = Callable[[str], Sequence[Token]]
+Decode = Callable[[Sequence[Token]], str]
+Prompt = Union[str, List[Dict]]
+
+
+def messages_to_string(messages: List[Dict[str, str]]) -> str:
+	"""Render messages only for token counting, never for model input."""
+	formatted_parts = [
+		f"<|im_start|>{message['role']}\n{message['content']}<|im_end|>"
+		for message in messages
+	]
+	formatted_parts.append("<|im_start|>assistant")
+	return "\n".join(formatted_parts)
+
+
+def count_message_tokens(messages: List[Dict], encode: Encode) -> int:
+	"""Count the tokenized representation of a chat prompt."""
+	return len(encode(messages_to_string(messages)))
+
+
+def truncate_messages_by_token(
+	messages: List[Dict], encode: Encode, decode: Decode, max_token_size: int
+) -> List[Dict]:
+	"""Fit a chat prompt in a token budget while preserving message structure."""
+	truncated_messages = [dict(message) for message in messages]
+	if count_message_tokens(truncated_messages, encode) <= max_token_size:
+		return truncated_messages
+
+	result = [{**message, "content": ""} for message in truncated_messages]
+	if count_message_tokens(result, encode) > max_token_size:
+		return result
+
+	for index, message in enumerate(truncated_messages):
+		content_tokens = encode(message.get("content", ""))
+		low, high = 0, len(content_tokens)
+		best_content = ""
+		while low <= high:
+			mid = (low + high) // 2
+			candidate = [dict(item) for item in result]
+			candidate[index]["content"] = decode(content_tokens[:mid])
+			if count_message_tokens(candidate, encode) <= max_token_size:
+				best_content = candidate[index]["content"]
+				low = mid + 1
+			else:
+				high = mid - 1
+
+		result[index]["content"] = best_content
+
+	return result
+
+
+def truncate_prompt_by_token(
+	prompt: Prompt, encode: Encode, decode: Decode, max_token_size: int
+) -> Prompt:
+	"""Truncate text or chat prompts while retaining a chat prompt's roles."""
+	if isinstance(prompt, list):
+		return truncate_messages_by_token(prompt, encode, decode, max_token_size)
+	tokens = encode(prompt)
+	return decode(tokens[:max_token_size])

{autorag-0.3.23 → autorag-0.3.25}/autorag/nodes/generator/llama_index_llm.py

@@ -55,7 +55,12 @@ class LlamaIndexLLM(BaseGenerator):
 						"`model` or `model_name` parameter must be provided for using huggingfacellm."
 					)
 			kwargs["tokenizer_name"] = kwargs["model_name"]
-		self.llm_instance: BaseLLM = llm_class(**pop_params(llm_class.__init__, kwargs))
+		llm_params = pop_params(llm_class.__init__, kwargs)
+		model_fields = getattr(llm_class, "model_fields", {})
+		for key in list(kwargs.keys()):
+			if key in model_fields:
+				llm_params[key] = kwargs.pop(key)
+		self.llm_instance: BaseLLM = llm_class(**llm_params)
 
 	def __del__(self):
 		super().__del__()

{autorag-0.3.23 → autorag-0.3.25}/autorag/nodes/generator/minimax_llm.py

@@ -8,6 +8,11 @@ from openai import AsyncOpenAI
 from tiktoken import Encoding
 
 from autorag.nodes.generator.base import BaseGenerator
+from autorag.nodes.generator.chat import (
+	count_message_tokens as count_message_tokens_with_encoder,
+	messages_to_string as render_messages_to_string,
+	truncate_prompt_by_token,
+)
 from autorag.utils.util import (
 	get_event_loop,
 	process_batch,
@@ -22,9 +27,10 @@ THINK_OPEN_TAG = "<think>"
 THINK_CLOSE_TAG = "</think>"
 
 MAX_TOKEN_DICT = {
-	"MiniMax-M2.7": 1_048_576,
-	"MiniMax-M2.7-highspeed": 1_048_576,
-	"MiniMax-M2.5": 1_048_576,
+	"MiniMax-M3": 1_000_000,
+	"MiniMax-M2.7": 204_800,
+	"MiniMax-M2.7-highspeed": 204_800,
+	"MiniMax-M2.5": 204_800,
 	"MiniMax-M2.5-highspeed": 204_800,
 }
 
@@ -38,7 +44,7 @@ class MiniMaxLLM(BaseGenerator):
 		It returns pseudo token ids and log probs since MiniMax does not support logprobs.
 
 		:param project_dir: The project directory.
-		:param llm: A model name for MiniMax. For example, ``MiniMax-M2.7`` or ``MiniMax-M2.5``.
+		:param llm: A model name for MiniMax. For example, ``MiniMax-M3`` or ``MiniMax-M2.7``.
 		:param batch: Batch size for API calls. Default is 16.
 		:param api_key: MiniMax API key. You can also set this to env variable ``MINIMAX_API_KEY``.
 		:param kwargs: Extra parameters for the MiniMax chat completion API.
@@ -172,7 +178,7 @@ class MiniMaxLLM(BaseGenerator):
 		)
 		answer = response.choices[0].message.content
 
-		# Strip thinking tags if present (MiniMax M2.5+ may include them)
+		# Strip thinking tags if present (MiniMax M2.7+ may include them)
 		if answer and THINK_OPEN_TAG in answer:
 			answer = strip_think_tags(answer)
 
@@ -185,63 +191,34 @@ class MiniMaxLLM(BaseGenerator):
 def truncate_by_token(
 	prompt: Union[str, List[Dict]], tokenizer: Encoding, max_token_size: int
 ) -> Union[str, List[Dict]]:
-	if isinstance(prompt, list):
-		return truncate_messages_by_token(prompt, tokenizer, max_token_size)
-	tokens = tokenizer.encode(prompt, allowed_special="all")
-	return tokenizer.decode(tokens[:max_token_size])
+	return truncate_prompt_by_token(
+		prompt,
+		lambda text: tokenizer.encode(text, allowed_special="all"),
+		tokenizer.decode,
+		max_token_size,
+	)
 
 
 def truncate_messages_by_token(
 	messages: List[Dict], tokenizer: Encoding, max_token_size: int
 ) -> List[Dict]:
-	truncated_messages = [dict(message) for message in messages]
-	if count_message_tokens(truncated_messages, tokenizer) <= max_token_size:
-		return truncated_messages
-
-	result: List[Dict] = []
-	for message in truncated_messages:
-		candidate = result + [message]
-		if count_message_tokens(candidate, tokenizer) <= max_token_size:
-			result.append(message)
-			continue
-
-		content_tokens = tokenizer.encode(
-			message.get("content", ""), allowed_special="all"
-		)
-		low, high = 0, len(content_tokens)
-		best_content = ""
-		while low <= high:
-			mid = (low + high) // 2
-			truncated_message = {
-				**message,
-				"content": tokenizer.decode(content_tokens[:mid]),
-			}
-			candidate = result + [truncated_message]
-			if count_message_tokens(candidate, tokenizer) <= max_token_size:
-				best_content = truncated_message["content"]
-				low = mid + 1
-			else:
-				high = mid - 1
-
-		if best_content or not result:
-			result.append({**message, "content": best_content})
-		break
-
-	return result
+	return truncate_prompt_by_token(
+		messages,
+		lambda text: tokenizer.encode(text, allowed_special="all"),
+		tokenizer.decode,
+		max_token_size,
+	)
 
 
 def count_message_tokens(messages: List[Dict], tokenizer: Encoding) -> int:
-	return len(tokenizer.encode(messages_to_string(messages), allowed_special="all"))
+	return count_message_tokens_with_encoder(
+		messages, lambda text: tokenizer.encode(text, allowed_special="all")
+	)
 
 
 def messages_to_string(messages: List[Dict[str, str]]) -> str:
-	"""Convert chat messages to string format for token counting."""
-	formatted_parts = [
-		f"<|im_start|>{message['role']}\n{message['content']}<|im_end|>"
-		for message in messages
-	]
-	formatted_parts.append("<|im_start|>assistant")
-	return "\n".join(formatted_parts)
+	"""Convert chat messages to a string for token counting compatibility."""
+	return render_messages_to_string(messages)
 
 
 def strip_think_tags(text: str) -> str: