AutoRAG 0.0.0__py3-none-any.whl

autorag/strategy.py

@@ -0,0 +1,165 @@
+import functools
+import time
+from typing import List, Iterable, Tuple, Any, Optional, Callable
+
+import numpy as np
+import pandas as pd
+
+
+def measure_speed(func, *args, **kwargs):
+	"""
+	Method for measuring execution speed of the function.
+	"""
+	start_time = time.time()
+	result = func(*args, **kwargs)
+	end_time = time.time()
+	return result, end_time - start_time
+
+
+def avoid_empty_result(return_index: List[int]):
+	"""
+	Decorator for avoiding empty results from the function.
+	When the func returns an empty result, it will return the origin results.
+	When the func returns a None, it will return the origin results.
+	When the return value is a tuple, it will check all the value or list is empty.
+	If so, it will return the origin results.
+	It keeps parameters at return_index of the function as the origin results.
+
+	:param return_index: The index of the result to be returned when there is no result.
+	:return: The origin results or the results from the function.
+	"""
+
+	def decorator_avoid_empty_result(func: Callable):
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs) -> List:
+			func_result = func(*args, **kwargs)
+			if isinstance(func_result, tuple):
+				# if all the results are empty, return the origin results.
+				if all([not bool(result) for result in func_result]):
+					return [args[index] for index in return_index]
+			if not bool(func_result):
+				return [args[index] for index in return_index]
+			else:
+				return func_result
+
+		return wrapper
+
+	return decorator_avoid_empty_result
+
+
+@avoid_empty_result([0, 3])
+def filter_by_threshold(results, value, threshold, metadatas=None) -> Tuple[List, List]:
+	"""
+	Filter results by value's threshold.
+
+	:param results: The result list to be filtered.
+	:param value: The value list to be filtered.
+	    It must have the same length with results.
+	:param threshold: The threshold value.
+	:param metadatas: The metadata of each result.
+	:return: Filtered list of results and filtered list of metadatas.
+	    Metadatas will be returned even if you did not give input metadatas.
+	:rtype: Tuple[List, List]
+	"""
+	if metadatas is None:
+		metadatas = [None] * len(results)
+	assert len(results) == len(value), "results and value must have the same length."
+	try:
+		filtered_results, _, filtered_metadatas = zip(
+			*filter(lambda x: x[1] <= threshold, zip(results, value, metadatas))
+		)
+	except ValueError:
+		return [], []
+	return list(filtered_results), list(filtered_metadatas)
+
+
+def validate_strategy_inputs(
+	results: List[pd.DataFrame],
+	columns: Iterable[str],
+	metadatas: Optional[List[Any]] = None,
+):
+	if metadatas is None:
+		metadatas = [None] * len(results)
+	assert len(results) == len(metadatas), (
+		"results and module_filename must have the same length."
+	)
+	assert all([isinstance(result, pd.DataFrame) for result in results]), (
+		"results must be pd.DataFrame."
+	)
+	assert all(
+		[column in result.columns for result in results for column in columns]
+	), "columns must be in the columns of results."
+	return results, columns, metadatas
+
+
+def select_best(
+	results: List[pd.DataFrame],
+	columns: Iterable[str],
+	metadatas: Optional[List[Any]] = None,
+	strategy_name: str = "mean",
+) -> Tuple[pd.DataFrame, Any]:
+	strategy_func_dict = {
+		"mean": select_best_average,
+		"rank": select_best_rr,
+		"normalize_mean": select_normalize_mean,
+	}
+	if strategy_name not in strategy_func_dict:
+		raise ValueError(
+			f"Input strategy name {strategy_name} is not in {strategy_func_dict.keys()}"
+		)
+
+	return strategy_func_dict[strategy_name](results, columns, metadatas)
+
+
+def select_best_average(
+	results: List[pd.DataFrame],
+	columns: Iterable[str],
+	metadatas: Optional[List[Any]] = None,
+) -> Tuple[pd.DataFrame, Any]:
+	"""
+	Select the best result by average value among given columns.
+
+	:param results: The list of results.
+	    Each result must be pd.DataFrame.
+	:param columns: Column names to be averaged.
+	    Standard to select the best result.
+	:param metadatas: The metadata of each result.
+	    It will select one metadata with the best result.
+	:return: The best result and the best metadata.
+	    The metadata will be returned even if you did not give input 'metadatas' parameter.
+	:rtype: Tuple[pd.DataFrame, Any]
+	"""
+	results, columns, metadatas = validate_strategy_inputs(results, columns, metadatas)
+	each_average = [df[columns].mean(axis=1).mean() for df in results]
+	best_index = each_average.index(max(each_average))
+	return results[best_index], metadatas[best_index]
+
+
+def select_best_rr(
+	results: List[pd.DataFrame],
+	columns: Iterable[str],
+	metadatas: Optional[List[Any]] = None,
+) -> Tuple[pd.DataFrame, Any]:
+	results, columns, metadatas = validate_strategy_inputs(results, columns, metadatas)
+	each_average_df = pd.DataFrame(
+		[df[columns].mean(axis=0).to_dict() for df in results]
+	)
+	rank_df = each_average_df.rank(ascending=False)
+	rr_df = rank_df.map(lambda x: 1 / x)
+	best_index = np.array(rr_df.sum(axis=1)).argmax()
+	return results[best_index], metadatas[best_index]
+
+
+def select_normalize_mean(
+	results: List[pd.DataFrame],
+	columns: Iterable[str],
+	metadatas: Optional[List[Any]] = None,
+) -> Tuple[pd.DataFrame, Any]:
+	results, columns, metadatas = validate_strategy_inputs(results, columns, metadatas)
+	each_mean_df = pd.DataFrame([df[columns].mean(axis=0).to_dict() for df in results])
+	normalized_means = (each_mean_df - each_mean_df.min()) / (
+		each_mean_df.max() - each_mean_df.min()
+	)
+	normalized_mean_sums = normalized_means.sum(axis=1)
+	best_index = normalized_mean_sums.argmax()
+	return results[best_index], metadatas[best_index]

autorag/support.py

@@ -0,0 +1,235 @@
+import importlib
+from typing import Callable, Dict
+
+
+def dynamically_find_function(key: str, target_dict: Dict) -> Callable:
+	if key in target_dict:
+		module_path, func_name = target_dict[key]
+		module = importlib.import_module(module_path)
+		func = getattr(module, func_name)
+		return func
+	else:
+		raise KeyError(f"Input module or node {key} is not supported.")
+
+
+def get_support_modules(module_name: str) -> Callable:
+	support_modules = {
+		# parse
+		"langchain_parse": ("autorag.data.parse", "langchain_parse"),
+		"clova": ("autorag.data.parse.clova", "clova_ocr"),
+		"llamaparse": ("autorag.data.parse.llamaparse", "llama_parse"),
+		"table_hybrid_parse": (
+			"autorag.data.parse.table_hybrid_parse",
+			"table_hybrid_parse",
+		),
+		# chunk
+		"llama_index_chunk": ("autorag.data.chunk", "llama_index_chunk"),
+		"langchain_chunk": ("autorag.data.chunk", "langchain_chunk"),
+		# query_expansion
+		"query_decompose": ("autorag.nodes.queryexpansion", "QueryDecompose"),
+		"hyde": ("autorag.nodes.queryexpansion", "HyDE"),
+		"pass_query_expansion": (
+			"autorag.nodes.queryexpansion",
+			"PassQueryExpansion",
+		),
+		"multi_query_expansion": (
+			"autorag.nodes.queryexpansion",
+			"MultiQueryExpansion",
+		),
+		"QueryDecompose": ("autorag.nodes.queryexpansion", "QueryDecompose"),
+		"HyDE": ("autorag.nodes.queryexpansion", "HyDE"),
+		"PassQueryExpansion": (
+			"autorag.nodes.queryexpansion",
+			"PassQueryExpansion",
+		),
+		"MultiQueryExpansion": (
+			"autorag.nodes.queryexpansion",
+			"MultiQueryExpansion",
+		),
+		# retrieval
+		"bm25": ("autorag.nodes.lexicalretrieval", "BM25"),
+		"BM25": ("autorag.nodes.lexicalretrieval", "BM25"),
+		"vectordb": ("autorag.nodes.semanticretrieval", "VectorDB"),
+		"VectorDB": ("autorag.nodes.semanticretrieval", "VectorDB"),
+		"hybrid_rrf": ("autorag.nodes.hybridretrieval", "HybridRRF"),
+		"HybridRRF": ("autorag.nodes.hybridretrieval", "HybridRRF"),
+		"hybrid_cc": ("autorag.nodes.hybridretrieval", "HybridCC"),
+		"HybridCC": ("autorag.nodes.hybridretrieval", "HybridCC"),
+		# passage_augmenter
+		"prev_next_augmenter": (
+			"autorag.nodes.passageaugmenter",
+			"PrevNextPassageAugmenter",
+		),
+		"PrevNextPassageAugmenter": (
+			"autorag.nodes.passageaugmenter",
+			"PrevNextPassageAugmenter",
+		),
+		"pass_passage_augmenter": (
+			"autorag.nodes.passageaugmenter",
+			"PassPassageAugmenter",
+		),
+		"PassPassageAugmenter": (
+			"autorag.nodes.passageaugmenter",
+			"PassPassageAugmenter",
+		),
+		# passage_reranker
+		"monot5": ("autorag.nodes.passagereranker", "MonoT5"),
+		"MonoT5": ("autorag.nodes.passagereranker", "MonoT5"),
+		"tart": ("autorag.nodes.passagereranker.tart", "Tart"),
+		"Tart": ("autorag.nodes.passagereranker.tart", "Tart"),
+		"upr": ("autorag.nodes.passagereranker", "Upr"),
+		"Upr": ("autorag.nodes.passagereranker", "Upr"),
+		"koreranker": ("autorag.nodes.passagereranker", "KoReranker"),
+		"KoReranker": ("autorag.nodes.passagereranker", "KoReranker"),
+		"pass_reranker": ("autorag.nodes.passagereranker", "PassReranker"),
+		"PassReranker": ("autorag.nodes.passagereranker", "PassReranker"),
+		"cohere_reranker": ("autorag.nodes.passagereranker", "CohereReranker"),
+		"CohereReranker": ("autorag.nodes.passagereranker", "CohereReranker"),
+		"rankgpt": ("autorag.nodes.passagereranker", "RankGPT"),
+		"RankGPT": ("autorag.nodes.passagereranker", "RankGPT"),
+		"jina_reranker": ("autorag.nodes.passagereranker", "JinaReranker"),
+		"JinaReranker": ("autorag.nodes.passagereranker", "JinaReranker"),
+		"colbert_reranker": ("autorag.nodes.passagereranker", "ColbertReranker"),
+		"ColbertReranker": ("autorag.nodes.passagereranker", "ColbertReranker"),
+		"sentence_transformer_reranker": (
+			"autorag.nodes.passagereranker",
+			"SentenceTransformerReranker",
+		),
+		"SentenceTransformerReranker": (
+			"autorag.nodes.passagereranker",
+			"SentenceTransformerReranker",
+		),
+		"flag_embedding_reranker": (
+			"autorag.nodes.passagereranker",
+			"FlagEmbeddingReranker",
+		),
+		"FlagEmbeddingReranker": (
+			"autorag.nodes.passagereranker",
+			"FlagEmbeddingReranker",
+		),
+		"flag_embedding_llm_reranker": (
+			"autorag.nodes.passagereranker",
+			"FlagEmbeddingLLMReranker",
+		),
+		"FlagEmbeddingLLMReranker": (
+			"autorag.nodes.passagereranker",
+			"FlagEmbeddingLLMReranker",
+		),
+		"time_reranker": ("autorag.nodes.passagereranker", "TimeReranker"),
+		"TimeReranker": ("autorag.nodes.passagereranker", "TimeReranker"),
+		"openvino_reranker": ("autorag.nodes.passagereranker", "OpenVINOReranker"),
+		"OpenVINOReranker": ("autorag.nodes.passagereranker", "OpenVINOReranker"),
+		"voyageai_reranker": ("autorag.nodes.passagereranker", "VoyageAIReranker"),
+		"VoyageAIReranker": ("autorag.nodes.passagereranker", "VoyageAIReranker"),
+		"mixedbreadai_reranker": (
+			"autorag.nodes.passagereranker",
+			"MixedbreadAIReranker",
+		),
+		"MixedbreadAIReranker": (
+			"autorag.nodes.passagereranker",
+			"MixedbreadAIReranker",
+		),
+		"flashrank_reranker": ("autorag.nodes.passagereranker", "FlashRankReranker"),
+		"FlashRankReranker": ("autorag.nodes.passagereranker", "FlashRankReranker"),
+		# passage_filter
+		"pass_passage_filter": ("autorag.nodes.passagefilter", "PassPassageFilter"),
+		"similarity_threshold_cutoff": (
+			"autorag.nodes.passagefilter",
+			"SimilarityThresholdCutoff",
+		),
+		"similarity_percentile_cutoff": (
+			"autorag.nodes.passagefilter",
+			"SimilarityPercentileCutoff",
+		),
+		"recency_filter": ("autorag.nodes.passagefilter", "RecencyFilter"),
+		"threshold_cutoff": ("autorag.nodes.passagefilter", "ThresholdCutoff"),
+		"percentile_cutoff": ("autorag.nodes.passagefilter", "PercentileCutoff"),
+		"PassPassageFilter": ("autorag.nodes.passagefilter", "PassPassageFilter"),
+		"SimilarityThresholdCutoff": (
+			"autorag.nodes.passagefilter",
+			"SimilarityThresholdCutoff",
+		),
+		"SimilarityPercentileCutoff": (
+			"autorag.nodes.passagefilter",
+			"SimilarityPercentileCutoff",
+		),
+		"RecencyFilter": ("autorag.nodes.passagefilter", "RecencyFilter"),
+		"ThresholdCutoff": ("autorag.nodes.passagefilter", "ThresholdCutoff"),
+		"PercentileCutoff": ("autorag.nodes.passagefilter", "PercentileCutoff"),
+		# passage_compressor
+		"tree_summarize": ("autorag.nodes.passagecompressor", "TreeSummarize"),
+		"pass_compressor": ("autorag.nodes.passagecompressor", "PassCompressor"),
+		"refine": ("autorag.nodes.passagecompressor", "Refine"),
+		"longllmlingua": ("autorag.nodes.passagecompressor", "LongLLMLingua"),
+		"TreeSummarize": ("autorag.nodes.passagecompressor", "TreeSummarize"),
+		"Refine": ("autorag.nodes.passagecompressor", "Refine"),
+		"LongLLMLingua": ("autorag.nodes.passagecompressor", "LongLLMLingua"),
+		"PassCompressor": ("autorag.nodes.passagecompressor", "PassCompressor"),
+		# prompt_maker
+		"fstring": ("autorag.nodes.promptmaker", "Fstring"),
+		"long_context_reorder": ("autorag.nodes.promptmaker", "LongContextReorder"),
+		"window_replacement": ("autorag.nodes.promptmaker", "WindowReplacement"),
+		"Fstring": ("autorag.nodes.promptmaker", "Fstring"),
+		"LongContextReorder": ("autorag.nodes.promptmaker", "LongContextReorder"),
+		"WindowReplacement": ("autorag.nodes.promptmaker", "WindowReplacement"),
+		"chat_fstring": ("autorag.nodes.promptmaker", "ChatFstring"),
+		"ChatFstring": ("autorag.nodes.promptmaker", "ChatFstring"),
+		# generator
+		"llama_index_llm": ("autorag.nodes.generator", "LlamaIndexLLM"),
+		"vllm": ("autorag.nodes.generator", "Vllm"),
+		"openai_llm": ("autorag.nodes.generator", "OpenAILLM"),
+		"vllm_api": ("autorag.nodes.generator", "VllmAPI"),
+		"LlamaIndexLLM": ("autorag.nodes.generator", "LlamaIndexLLM"),
+		"Vllm": ("autorag.nodes.generator", "Vllm"),
+		"OpenAILLM": ("autorag.nodes.generator", "OpenAILLM"),
+		"VllmAPI": ("autorag.nodes.generator", "VllmAPI"),
+	}
+	return dynamically_find_function(module_name, support_modules)
+
+
+def get_support_nodes(node_name: str) -> Callable:
+	support_nodes = {
+		"query_expansion": (
+			"autorag.nodes.queryexpansion.run",
+			"run_query_expansion_node",
+		),
+		"semantic_retrieval": (
+			"autorag.nodes.semanticretrieval.run",
+			"run_semantic_retrieval_node",
+		),
+		"dense_retrieval": (
+			"autorag.nodes.semanticretrieval.run",
+			"run_semantic_retrieval_node",
+		),
+		"lexical_retrieval": (
+			"autorag.nodes.lexicalretrieval.run",
+			"run_lexical_retrieval_node",
+		),
+		"sparse_retrieval": (
+			"autorag.nodes.lexicalretrieval.run",
+			"run_lexical_retrieval_node",
+		),
+		"hybrid_retrieval": (
+			"autorag.nodes.hybridretrieval.run",
+			"run_hybrid_retrieval_node",
+		),
+		"generator": ("autorag.nodes.generator.run", "run_generator_node"),
+		"prompt_maker": ("autorag.nodes.promptmaker.run", "run_prompt_maker_node"),
+		"passage_filter": (
+			"autorag.nodes.passagefilter.run",
+			"run_passage_filter_node",
+		),
+		"passage_compressor": (
+			"autorag.nodes.passagecompressor.run",
+			"run_passage_compressor_node",
+		),
+		"passage_reranker": (
+			"autorag.nodes.passagereranker.run",
+			"run_passage_reranker_node",
+		),
+		"passage_augmenter": (
+			"autorag.nodes.passageaugmenter.run",
+			"run_passage_augmenter_node",
+		),
+	}
+	return dynamically_find_function(node_name, support_nodes)

autorag/utils/__init__.py

@@ -0,0 +1,8 @@
+from .preprocess import (
+	validate_qa_dataset,
+	validate_corpus_dataset,
+	cast_qa_dataset,
+	cast_corpus_dataset,
+	validate_qa_from_corpus_dataset,
+)
+from .util import fetch_contents, result_to_dataframe, sort_by_scores

autorag/utils/cast.py

@@ -0,0 +1,45 @@
+def cast_retrieve_infos(previous_result):
+	return {
+		"retrieved_contents": cast_retrieved_contents(previous_result),
+		"retrieved_ids": cast_retrieved_ids(previous_result),
+		"retrieve_scores": cast_retrieve_scores(previous_result),
+	}
+
+
+def cast_retrieved_contents(previous_result):
+	if "retrieved_contents" in previous_result.columns:
+		return previous_result["retrieved_contents"].tolist()
+	elif "retrieved_contents_semantic" in previous_result.columns:
+		return previous_result["retrieved_contents_semantic"].tolist()
+	elif "retrieved_contents_lexical" in previous_result.columns:
+		return previous_result["retrieved_contents_lexical"].tolist()
+	else:
+		raise ValueError(
+			"previous_result must contain either 'retrieved_contents', 'retrieved_contents_semantic', or 'retrieved_contents_lexical' columns."
+		)
+
+
+def cast_retrieved_ids(previous_result):
+	if "retrieved_ids" in previous_result.columns:
+		return previous_result["retrieved_ids"].tolist()
+	elif "retrieved_ids_semantic" in previous_result.columns:
+		return previous_result["retrieved_ids_semantic"].tolist()
+	elif "retrieved_ids_lexical" in previous_result.columns:
+		return previous_result["retrieved_ids_lexical"].tolist()
+	else:
+		raise ValueError(
+			"previous_result must contain either 'retrieved_ids', 'retrieved_ids_semantic', or 'retrieved_ids_lexical' columns."
+		)
+
+
+def cast_retrieve_scores(previous_result):
+	if "retrieve_scores" in previous_result.columns:
+		return previous_result["retrieve_scores"].tolist()
+	elif "retrieve_scores_semantic" in previous_result.columns:
+		return previous_result["retrieve_scores_semantic"].tolist()
+	elif "retrieve_scores_lexical" in previous_result.columns:
+		return previous_result["retrieve_scores_lexical"].tolist()
+	else:
+		raise ValueError(
+			"previous_result must contain either 'retrieve_scores', 'retrieve_scores_semantic', or 'retrieve_scores_lexical' columns."
+		)

autorag/utils/preprocess.py

@@ -0,0 +1,149 @@
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+from autorag.utils.util import preprocess_text
+
+
+def validate_qa_dataset(df: pd.DataFrame):
+	columns = ["qid", "query", "retrieval_gt", "generation_gt"]
+	assert set(columns).issubset(df.columns), (
+		f"df must have columns {columns}, but got {df.columns}"
+	)
+
+
+def validate_corpus_dataset(df: pd.DataFrame):
+	columns = ["doc_id", "contents", "metadata"]
+	assert set(columns).issubset(df.columns), (
+		f"df must have columns {columns}, but got {df.columns}"
+	)
+
+
+def cast_qa_dataset(df: pd.DataFrame):
+	def cast_retrieval_gt(gt):
+		if isinstance(gt, str):
+			return [[gt]]
+		elif isinstance(gt, list):
+			if isinstance(gt[0], str):
+				return [gt]
+			elif isinstance(gt[0], list):
+				return gt
+			elif isinstance(gt[0], np.ndarray):
+				return cast_retrieval_gt(list(map(lambda x: x.tolist(), gt)))
+			else:
+				raise ValueError(
+					f"retrieval_gt must be str or list, but got {type(gt[0])}"
+				)
+		elif isinstance(gt, np.ndarray):
+			return cast_retrieval_gt(gt.tolist())
+		else:
+			raise ValueError(f"retrieval_gt must be str or list, but got {type(gt)}")
+
+	def cast_generation_gt(gt):
+		if isinstance(gt, str):
+			return [gt]
+		elif isinstance(gt, list):
+			return gt
+		elif isinstance(gt, np.ndarray):
+			return cast_generation_gt(gt.tolist())
+		else:
+			raise ValueError(f"generation_gt must be str or list, but got {type(gt)}")
+
+	df = df.reset_index(drop=True)
+	validate_qa_dataset(df)
+	assert df["qid"].apply(lambda x: isinstance(x, str)).sum() == len(df), (
+		"qid must be string type."
+	)
+	assert df["query"].apply(lambda x: isinstance(x, str)).sum() == len(df), (
+		"query must be string type."
+	)
+	df["retrieval_gt"] = df["retrieval_gt"].apply(cast_retrieval_gt)
+	df["generation_gt"] = df["generation_gt"].apply(cast_generation_gt)
+	df["query"] = df["query"].apply(preprocess_text)
+	df["generation_gt"] = df["generation_gt"].apply(
+		lambda x: list(map(preprocess_text, x))
+	)
+	return df
+
+
+def cast_corpus_dataset(df: pd.DataFrame):
+	df = df.reset_index(drop=True)
+	validate_corpus_dataset(df)
+
+	# drop rows that have empty contents
+	df = df[~df["contents"].apply(lambda x: x is None or x.isspace())]
+
+	def make_datetime_metadata(x):
+		if x is None or x == {}:
+			return {"last_modified_datetime": datetime.now()}
+		elif x.get("last_modified_datetime") is None:
+			return {**x, "last_modified_datetime": datetime.now()}
+		else:
+			return x
+
+	df["metadata"] = df["metadata"].apply(make_datetime_metadata)
+
+	# check every metadata have a datetime key
+	assert sum(
+		df["metadata"].apply(lambda x: x.get("last_modified_datetime") is not None)
+	) == len(df), "Every metadata must have a datetime key."
+
+	def make_prev_next_id_metadata(x, id_type: str):
+		if x is None or x == {}:
+			return {id_type: None}
+		elif x.get(id_type) is None:
+			return {**x, id_type: None}
+		else:
+			return x
+
+	df["metadata"] = df["metadata"].apply(
+		lambda x: make_prev_next_id_metadata(x, "prev_id")
+	)
+	df["metadata"] = df["metadata"].apply(
+		lambda x: make_prev_next_id_metadata(x, "next_id")
+	)
+
+	df["contents"] = df["contents"].apply(preprocess_text)
+
+	def normalize_unicode_metadata(metadata: dict):
+		result = {}
+		for key, value in metadata.items():
+			if isinstance(value, str):
+				result[key] = preprocess_text(value)
+			else:
+				result[key] = value
+		return result
+
+	df["metadata"] = df["metadata"].apply(normalize_unicode_metadata)
+
+	# check every metadata have a prev_id, next_id key
+	assert all("prev_id" in metadata for metadata in df["metadata"]), (
+		"Every metadata must have a prev_id key."
+	)
+	assert all("next_id" in metadata for metadata in df["metadata"]), (
+		"Every metadata must have a next_id key."
+	)
+
+	return df
+
+
+def validate_qa_from_corpus_dataset(qa_df: pd.DataFrame, corpus_df: pd.DataFrame):
+	qa_ids = []
+	for retrieval_gt in qa_df["retrieval_gt"].tolist():
+		if isinstance(retrieval_gt, list) and (
+			retrieval_gt[0] != [] or any(bool(g) is True for g in retrieval_gt)
+		):
+			for gt in retrieval_gt:
+				qa_ids.extend(gt)
+		elif isinstance(retrieval_gt, np.ndarray) and retrieval_gt[0].size > 0:
+			for gt in retrieval_gt:
+				qa_ids.extend(gt)
+
+	no_exist_ids = list(
+		filter(lambda qa_id: corpus_df[corpus_df["doc_id"] == qa_id].empty, qa_ids)
+	)
+
+	assert len(no_exist_ids) == 0, (
+		f"{len(no_exist_ids)} doc_ids in retrieval_gt do not exist in corpus_df."
+	)