AutoRAG 0.0.0__py3-none-any.whl

autorag/evaluation/metric/util.py

@@ -0,0 +1,88 @@
+import functools
+from typing import List
+
+import numpy as np
+
+from autorag.schema.metricinput import MetricInput
+from autorag.utils.util import convert_inputs_to_list
+
+
+def calculate_cosine_similarity(a, b):
+	dot_product = np.dot(a, b)
+	norm_a = np.linalg.norm(a)
+	norm_b = np.linalg.norm(b)
+	similarity = dot_product / (norm_a * norm_b)
+	return similarity
+
+
+def calculate_l2_distance(a, b):
+	return np.linalg.norm(a - b)
+
+
+def calculate_inner_product(a, b):
+	return np.dot(a, b)
+
+
+def autorag_metric(fields_to_check: List[str]):
+	def decorator_autorag_metric(func):
+		@functools.wraps(func)
+		@convert_inputs_to_list
+		def wrapper(metric_inputs: List[MetricInput], **kwargs) -> List[float]:
+			"""
+			Use 'for loop' to run each metric input.
+			Put the single metric input into the metric function.
+
+			:param metric_inputs: A list MetricInput schema for AutoRAG metric.
+			:param kwargs: The additional arguments for metric function.
+			:return: A list of computed metric scores.
+			"""
+			results = []
+			for metric_input in metric_inputs:
+				if metric_input.is_fields_notnone(fields_to_check=fields_to_check):
+					results.append(func(metric_input, **kwargs))
+				else:
+					results.append(None)
+			return results
+
+		return wrapper
+
+	return decorator_autorag_metric
+
+
+def autorag_metric_loop(fields_to_check: List[str]):
+	def decorator_autorag_generation_metric(func):
+		@functools.wraps(func)
+		@convert_inputs_to_list
+		def wrapper(metric_inputs: List[MetricInput], **kwargs) -> List[float]:
+			"""
+			Put the list of metric inputs into the metric function.
+
+			:param metric_inputs: A list MetricInput schema for AutoRAG metric.
+			:param kwargs: The additional arguments for metric function.
+			:return: A list of computed metric scores.
+			"""
+			bool_list = [
+				metric_input.is_fields_notnone(fields_to_check=fields_to_check)
+				for metric_input in metric_inputs
+			]
+			valid_inputs = [
+				metric_input
+				for metric_input, is_valid in zip(metric_inputs, bool_list)
+				if is_valid
+			]
+
+			results = [None] * len(metric_inputs)
+			if valid_inputs:
+				processed_valid = func(valid_inputs, **kwargs)
+
+				valid_index = 0
+				for i, is_valid in enumerate(bool_list):
+					if is_valid:
+						results[i] = processed_valid[valid_index]
+						valid_index += 1
+
+			return results
+
+		return wrapper
+
+	return decorator_autorag_generation_metric

autorag/evaluation/retrieval.py

@@ -0,0 +1,83 @@
+import functools
+import warnings
+from typing import List, Callable, Any, Tuple, Union, Dict
+
+import pandas as pd
+
+from autorag.evaluation.metric import (
+	retrieval_recall,
+	retrieval_precision,
+	retrieval_f1,
+	retrieval_ndcg,
+	retrieval_mrr,
+	retrieval_map,
+)
+from autorag.evaluation.util import cast_metrics
+from autorag.schema.metricinput import MetricInput
+
+RETRIEVAL_METRIC_FUNC_DICT = {
+	func.__name__: func
+	for func in [
+		retrieval_recall,
+		retrieval_precision,
+		retrieval_f1,
+		retrieval_ndcg,
+		retrieval_mrr,
+		retrieval_map,
+	]
+}
+
+
+def evaluate_retrieval(
+	metric_inputs: List[MetricInput],
+	metrics: Union[List[str], List[Dict]],
+):
+	def decorator_evaluate_retrieval(
+		func: Callable[
+			[Any], Tuple[List[List[str]], List[List[str]], List[List[float]]]
+		],
+	):
+		"""
+		Decorator for evaluating retrieval results.
+		You can use this decorator to any method that returns (contents, scores, ids),
+		which is the output of conventional retrieval modules.
+
+		:param func: Must return (contents, scores, ids)
+		:return: wrapper function that returns pd.DataFrame, which is the evaluation result.
+		"""
+
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs) -> pd.DataFrame:
+			contents, pred_ids, scores = func(*args, **kwargs)
+			for metric_input, pred_id in zip(metric_inputs, pred_ids):
+				metric_input.retrieved_ids = pred_id
+
+			metric_scores = {}
+			metric_names, metric_params = cast_metrics(metrics)
+
+			for metric_name, metric_param in zip(metric_names, metric_params):
+				if metric_name in RETRIEVAL_METRIC_FUNC_DICT:
+					metric_func = RETRIEVAL_METRIC_FUNC_DICT[metric_name]
+					metric_scores[metric_name] = metric_func(
+						metric_inputs=metric_inputs, **metric_param
+					)
+				else:
+					warnings.warn(
+						f"metric {metric_name} is not in supported metrics: {RETRIEVAL_METRIC_FUNC_DICT.keys()}"
+						f"{metric_name} will be ignored."
+					)
+
+			metric_result_df = pd.DataFrame(metric_scores)
+			execution_result_df = pd.DataFrame(
+				{
+					"retrieved_contents": contents,
+					"retrieved_ids": pred_ids,
+					"retrieve_scores": scores,
+				}
+			)
+			result_df = pd.concat([execution_result_df, metric_result_df], axis=1)
+			return result_df
+
+		return wrapper
+
+	return decorator_evaluate_retrieval

autorag/evaluation/retrieval_contents.py

@@ -0,0 +1,65 @@
+import functools
+from typing import List, Callable, Any, Tuple
+
+import pandas as pd
+
+from autorag.evaluation.metric import (
+	retrieval_token_f1,
+	retrieval_token_precision,
+	retrieval_token_recall,
+)
+from autorag.schema.metricinput import MetricInput
+
+
+def evaluate_retrieval_contents(metric_inputs: List[MetricInput], metrics: List[str]):
+	def decorator_evaluate_retrieval_contents(
+		func: Callable[
+			[Any], Tuple[List[List[str]], List[List[str]], List[List[float]]]
+		],
+	):
+		"""
+		Decorator for evaluating retrieval contents.
+		You can use this decorator to any method that returns (contents, scores, ids),
+		which is the output of conventional retrieval modules.
+
+		:param func: Must return (contents, scores, ids)
+		:return: pd.DataFrame, which is the evaluation result and function result.
+		"""
+
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs) -> pd.DataFrame:
+			contents, pred_ids, scores = func(*args, **kwargs)
+			metric_funcs = {
+				retrieval_token_recall.__name__: retrieval_token_recall,
+				retrieval_token_precision.__name__: retrieval_token_precision,
+				retrieval_token_f1.__name__: retrieval_token_f1,
+			}
+			for metric_input, content in zip(metric_inputs, contents):
+				metric_input.retrieved_contents = content
+
+			metrics_scores = {}
+			for metric in metrics:
+				if metric not in metric_funcs:
+					raise ValueError(
+						f"metric {metric} is not in supported metrics: {metric_funcs.keys()}"
+					)
+				else:
+					metric_func = metric_funcs[metric]
+					# Extract each required field from all payloads
+					metric_scores = metric_func(metric_inputs=metric_inputs)
+					metrics_scores[metric] = metric_scores
+
+			metric_result_df = pd.DataFrame(metrics_scores)
+			execution_result_df = pd.DataFrame(
+				{
+					"retrieved_contents": contents,
+					"retrieved_ids": pred_ids,
+					"retrieve_scores": scores,
+				}
+			)
+			result_df = pd.concat([execution_result_df, metric_result_df], axis=1)
+			return result_df
+
+		return wrapper
+
+	return decorator_evaluate_retrieval_contents

autorag/evaluation/util.py

@@ -0,0 +1,43 @@
+from copy import deepcopy
+from typing import Union, List, Dict, Tuple, Any
+
+from autorag.embedding.base import EmbeddingModel
+
+
+def cast_metrics(
+	metrics: Union[List[str], List[Dict]],
+) -> Tuple[List[str], List[Dict[str, Any]]]:
+	"""
+	 Turn metrics to list of metric names and parameter list.
+
+	:param metrics: List of string or dictionary.
+	:return: The list of metric names and dictionary list of metric parameters.
+	"""
+	metrics_copy = deepcopy(metrics)
+	if not isinstance(metrics_copy, list):
+		raise ValueError("metrics must be a list of string or dictionary.")
+	if isinstance(metrics_copy[0], str):
+		return metrics_copy, [{} for _ in metrics_copy]
+	elif isinstance(metrics_copy[0], dict):
+		# pop 'metric_name' key from dictionary
+		metric_names = list(map(lambda x: x.pop("metric_name"), metrics_copy))
+		metric_params = [
+			dict(
+				map(
+					lambda x, y: cast_embedding_model(x, y),
+					metric.keys(),
+					metric.values(),
+				)
+			)
+			for metric in metrics_copy
+		]
+		return metric_names, metric_params
+	else:
+		raise ValueError("metrics must be a list of string or dictionary.")
+
+
+def cast_embedding_model(key, value):
+	if key == "embedding_model":
+		return key, EmbeddingModel.load(value)()
+	else:
+		return key, value