AutoRAG 0.2.9__tar.gz → 0.2.10__tar.gz

{autorag-0.2.9 → autorag-0.2.10}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.9
+Version: 0.2.10
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

{autorag-0.2.9 → autorag-0.2.10}/AutoRAG.egg-info/SOURCES.txt

@@ -116,9 +116,7 @@ autorag/nodes/retrieval/__init__.py
 autorag/nodes/retrieval/base.py
 autorag/nodes/retrieval/bm25.py
 autorag/nodes/retrieval/hybrid_cc.py
-autorag/nodes/retrieval/hybrid_dbsf.py
 autorag/nodes/retrieval/hybrid_rrf.py
-autorag/nodes/retrieval/hybrid_rsf.py
 autorag/nodes/retrieval/run.py
 autorag/nodes/retrieval/vectordb.py
 autorag/schema/__init__.py
@@ -235,15 +233,12 @@ docs/source/nodes/query_expansion/query_decompose.md
 docs/source/nodes/query_expansion/query_expansion.md
 docs/source/nodes/retrieval/bm25.md
 docs/source/nodes/retrieval/hybrid_cc.md
-docs/source/nodes/retrieval/hybrid_dbsf.md
 docs/source/nodes/retrieval/hybrid_rrf.md
-docs/source/nodes/retrieval/hybrid_rsf.md
 docs/source/nodes/retrieval/retrieval.md
 docs/source/nodes/retrieval/vectordb.md
 docs/source/optimization/custom_config.md
 docs/source/optimization/folder_structure.md
 docs/source/optimization/optimization.md
-docs/source/optimization/sample_full_config.yaml
 docs/source/optimization/strategies.md
 docs/source/roadmap/modular_rag.md
 sample_config/compact_local.yaml
@@ -336,9 +331,7 @@ tests/autorag/nodes/queryexpansion/test_query_expansion_run.py
 tests/autorag/nodes/retrieval/test_bm25.py
 tests/autorag/nodes/retrieval/test_hybrid_base.py
 tests/autorag/nodes/retrieval/test_hybrid_cc.py
-tests/autorag/nodes/retrieval/test_hybrid_dbsf.py
 tests/autorag/nodes/retrieval/test_hybrid_rrf.py
-tests/autorag/nodes/retrieval/test_hybrid_rsf.py
 tests/autorag/nodes/retrieval/test_retrieval_base.py
 tests/autorag/nodes/retrieval/test_run_retrieval_node.py
 tests/autorag/nodes/retrieval/test_vectordb.py

{autorag-0.2.9 → autorag-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.9
+Version: 0.2.10
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

autorag-0.2.10/autorag/VERSION

@@ -0,0 +1 @@
+0.2.10

{autorag-0.2.9 → autorag-0.2.10}/autorag/nodes/retrieval/__init__.py

@@ -1,7 +1,5 @@
 from .base import retrieval_node
 from .bm25 import bm25
 from .hybrid_cc import hybrid_cc
-from .hybrid_dbsf import hybrid_dbsf
 from .hybrid_rrf import hybrid_rrf
-from .hybrid_rsf import hybrid_rsf
 from .vectordb import vectordb

{autorag-0.2.9 → autorag-0.2.10}/autorag/nodes/retrieval/base.py

@@ -32,7 +32,6 @@ def retrieval_node(func):
             project_dir: Union[str, Path],
             previous_result: pd.DataFrame,
             **kwargs) -> Tuple[List[List[str]], List[List[str]], List[List[float]]]:
-        logger.info(f"Running retrieval node - {func.__name__} module...")
         validate_qa_dataset(previous_result)
         resources_dir = os.path.join(project_dir, "resources")
         data_dir = os.path.join(project_dir, "data")
@@ -75,10 +74,10 @@ def retrieval_node(func):
             del embedding_model
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
-        elif func.__name__ in ["hybrid_rrf", "hybrid_cc", "hybrid_rsf", "hybrid_dbsf"]:
-            if 'ids' in kwargs and 'scores' in kwargs:
+        elif func.__name__ in ["hybrid_rrf", "hybrid_cc"]:
+            if 'ids' in kwargs and 'scores' in kwargs:  # ordinary run_evaluate
                 ids, scores = func(**kwargs)
-            else:
+            else:  # => for Runner.run
                 if not ('target_modules' in kwargs and 'target_module_params' in kwargs):
                     raise ValueError(
                         f"If there are no ids and scores specified, target_modules and target_module_params must be specified for using {func.__name__}.")

autorag-0.2.10/autorag/nodes/retrieval/hybrid_cc.py

@@ -0,0 +1,137 @@
+from typing import Tuple, List
+
+import numpy as np
+import pandas as pd
+
+from autorag.nodes.retrieval import retrieval_node
+
+
+def normalize_mm(scores: List[str], fixed_min_value: float = 0):
+    arr = np.array(scores)
+    max_value = np.max(arr)
+    min_value = np.min(arr)
+    norm_score = (arr - min_value) / (max_value - min_value)
+    return norm_score
+
+
+def normalize_tmm(scores: List[str], fixed_min_value: float):
+    arr = np.array(scores)
+    max_value = np.max(arr)
+    norm_score = (arr - fixed_min_value) / (max_value - fixed_min_value)
+    return norm_score
+
+
+def normalize_z(scores: List[str], fixed_min_value: float = 0):
+    arr = np.array(scores)
+    mean_value = np.mean(arr)
+    std_value = np.std(arr)
+    norm_score = (arr - mean_value) / std_value
+    return norm_score
+
+
+def normalize_dbsf(scores: List[str], fixed_min_value: float = 0):
+    arr = np.array(scores)
+    mean_value = np.mean(arr)
+    std_value = np.std(arr)
+    min_value = mean_value - 3 * std_value
+    max_value = mean_value + 3 * std_value
+    norm_score = (arr - min_value) / (max_value - min_value)
+    return norm_score
+
+
+normalize_method_dict = {
+    'mm': normalize_mm,
+    'tmm': normalize_tmm,
+    'z': normalize_z,
+    'dbsf': normalize_dbsf,
+}
+
+
+@retrieval_node
+def hybrid_cc(
+        ids: Tuple,
+        scores: Tuple,
+        top_k: int,
+        weight: float,
+        normalize_method: str = 'mm',
+        semantic_theoretical_min_value: float = -1.0,
+        lexical_theoretical_min_value: float = 0.0,
+) -> Tuple[List[List[str]], List[List[float]]]:
+    """
+    Hybrid CC function.
+    CC (convex combination) is a method to fuse lexical and semantic retrieval results.
+    It is a method that first normalizes the scores of each retrieval result,
+    and then combines them with the given weights.
+    It is uniquer than other retrieval modules, because it does not really execute retrieval,
+    but just fuse the results of other retrieval functions.
+    So you have to run more than two retrieval modules before running this function.
+    And collect ids and scores result from each retrieval module.
+    Make it as tuple and input it to this function.
+
+    :param ids: The tuple of ids that you want to fuse.
+        The length of this must be the same as the length of scores.
+        The semantic retrieval ids must be the first index.
+    :param scores: The retrieve scores that you want to fuse.
+        The length of this must be the same as the length of ids.
+        The semantic retrieval scores must be the first index.
+    :param top_k: The number of passages to be retrieved.
+    :param normalize_method: The normalization method to use.
+      There are some normalization method that you can use at the hybrid cc method.
+      AutoRAG support following.
+        - `mm`: Min-max scaling
+        - `tmm`: Theoretical min-max scaling
+        - `z`: z-score normalization
+        - `dbsf`: 3-sigma normalization
+    :param weight: The weight value. If the weight is 1.0, it means the
+      weight to the semantic module will be 1.0 and weight to the lexical module will be 0.0.
+    :param semantic_theoretical_min_value: This value used by `tmm` normalization method. You can set the
+        theoretical minimum value by yourself. Default is -1.
+    :param lexical_theoretical_min_value: This value used by `tmm` normalization method. You can set the
+        theoretical minimum value by yourself. Default is 0.
+    :return: The tuple of ids and fused scores that fused by CC. Plus, the third element is selected weight value.
+    """
+    assert len(ids) == len(scores), "The length of ids and scores must be the same."
+    assert len(ids) > 1, "You must input more than one retrieval results."
+    assert top_k > 0, "top_k must be greater than 0."
+    assert weight >= 0, "The weight must be greater than 0."
+    assert weight <= 1, "The weight must be less than 1."
+
+    df = pd.DataFrame({
+        'semantic_ids': ids[0],
+        'lexical_ids': ids[1],
+        'semantic_score': scores[0],
+        'lexical_score': scores[1],
+    })
+
+    def cc_pure_apply(row):
+        return fuse_per_query(row['semantic_ids'], row['lexical_ids'],
+                              row['semantic_score'], row['lexical_score'],
+                              normalize_method=normalize_method,
+                              weight=weight, top_k=top_k,
+                              semantic_theoretical_min_value=semantic_theoretical_min_value,
+                              lexical_theoretical_min_value=lexical_theoretical_min_value)
+
+    # fixed weight
+    df[['cc_id', 'cc_score']] = df.apply(lambda row: cc_pure_apply(row), axis=1,
+                                         result_type='expand')
+    return df['cc_id'].tolist(), df['cc_score'].tolist()
+
+
+def fuse_per_query(semantic_ids: List[str], lexical_ids: List[str],
+                   semantic_scores: List[float], lexical_scores: List[float],
+                   normalize_method: str,
+                   weight: float,
+                   top_k: int,
+                   semantic_theoretical_min_value: float,
+                   lexical_theoretical_min_value: float):
+    normalize_func = normalize_method_dict[normalize_method]
+    norm_semantic_scores = normalize_func(semantic_scores, semantic_theoretical_min_value)
+    norm_lexical_scores = normalize_func(lexical_scores, lexical_theoretical_min_value)
+    ids = [semantic_ids, lexical_ids]
+    scores = [norm_semantic_scores, norm_lexical_scores]
+    df = pd.concat([pd.Series(dict(zip(_id, score))) for _id, score in zip(ids, scores)], axis=1)
+    df.columns = ['semantic', 'lexical']
+    df = df.fillna(0)
+    df['weighted_sum'] = df.mul((weight, 1.0 - weight)).sum(axis=1)
+    df = df.sort_values(by='weighted_sum', ascending=False)
+    return df.index.tolist()[:top_k], df['weighted_sum'][:top_k].tolist()

{autorag-0.2.9 → autorag-0.2.10}/autorag/nodes/retrieval/hybrid_rrf.py

@@ -10,7 +10,8 @@ def hybrid_rrf(
         ids: Tuple,
         scores: Tuple,
         top_k: int,
-        rrf_k: int = 60) -> Tuple[List[List[str]], List[List[float]]]:
+        weight: int = 60,
+        rrf_k: int = -1, ) -> Tuple[List[List[str]], List[List[float]]]:
     """
     Hybrid RRF function.
     RRF (Rank Reciprocal Fusion) is a method to fuse multiple retrieval results.
@@ -27,15 +28,23 @@ def hybrid_rrf(
     :param scores: The retrieve scores that you want to fuse.
         The length of this must be the same as the length of ids.
     :param top_k: The number of passages to be retrieved.
-    :param rrf_k: Hyperparameter for RRF.
+    :param weight: Hyperparameter for RRF.
+        It was originally rrf_k value.
         Default is 60.
         For more information, please visit our documentation.
+    :param rrf_k: (Deprecated) Hyperparameter for RRF.
+        It was originally rrf_k value. Will remove at further version.
     :return: The tuple of ids and fused scores that fused by RRF.
     """
     assert len(ids) == len(scores), "The length of ids and scores must be the same."
     assert len(ids) > 1, "You must input more than one retrieval results."
     assert top_k > 0, "top_k must be greater than 0."
-    assert rrf_k > 0, "rrf_k must be greater than 0."
+    assert weight > 0, "rrf_k must be greater than 0."
+
+    if rrf_k != -1:
+        weight = int(rrf_k)
+    else:
+        weight = int(weight)
 
     id_df = pd.DataFrame({f'id_{i}': id_list for i, id_list in enumerate(ids)})
     score_df = pd.DataFrame({f'score_{i}': score_list for i, score_list in enumerate(scores)})
@@ -44,9 +53,9 @@ def hybrid_rrf(
     def rrf_pure_apply(row):
         ids_tuple = tuple(row[[f'id_{i}' for i in range(len(ids))]].values)
         scores_tuple = tuple(row[[f'score_{i}' for i in range(len(scores))]].values)
-        return pd.Series(rrf_pure(ids_tuple, scores_tuple, rrf_k, top_k))
+        return pd.Series(rrf_pure(ids_tuple, scores_tuple, weight, top_k))
 
-    df[['rrf_id', 'rrf_score']] = df.swifter.apply(rrf_pure_apply, axis=1)
+    df[['rrf_id', 'rrf_score']] = df.apply(rrf_pure_apply, axis=1)
     return df['rrf_id'].tolist(), df['rrf_score'].tolist()
 
 

autorag-0.2.10/autorag/nodes/retrieval/run.py

@@ -0,0 +1,285 @@
+import logging
+import os
+import pathlib
+from typing import List, Callable, Dict, Tuple
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from autorag.evaluation import evaluate_retrieval
+from autorag.strategy import measure_speed, filter_by_threshold, select_best
+
+logger = logging.getLogger("AutoRAG")
+
+semantic_module_names = ['vectordb']
+lexical_module_names = ['bm25']
+hybrid_module_names = ['hybrid_rrf', 'hybrid_cc']
+
+
+def run_retrieval_node(modules: List[Callable],
+                       module_params: List[Dict],
+                       previous_result: pd.DataFrame,
+                       node_line_dir: str,
+                       strategies: Dict,
+                       ) -> pd.DataFrame:
+    """
+    Run evaluation and select the best module among retrieval node results.
+
+    :param modules: Retrieval modules to run.
+    :param module_params: Retrieval module parameters.
+    :param previous_result: Previous result dataframe.
+        Could be query expansion's best result or qa data.
+    :param node_line_dir: This node line's directory.
+    :param strategies: Strategies for retrieval node.
+    :return: The best result dataframe.
+        It contains previous result columns and retrieval node's result columns.
+    """
+    if not os.path.exists(node_line_dir):
+        os.makedirs(node_line_dir)
+    project_dir = pathlib.PurePath(node_line_dir).parent.parent
+    qa_df = pd.read_parquet(os.path.join(project_dir, "data", "qa.parquet"), engine='pyarrow')
+    retrieval_gt = qa_df['retrieval_gt'].tolist()
+    retrieval_gt = [[[str(uuid) for uuid in sub_array] if sub_array.size > 0 else [] for sub_array in inner_array]
+                    for inner_array in retrieval_gt]
+
+    save_dir = os.path.join(node_line_dir, "retrieval")  # node name
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+
+    def run(input_modules, input_module_params) -> Tuple[List[pd.DataFrame], List]:
+        """
+            Run input modules and parameters.
+
+            :param input_modules: Input modules
+            :param input_module_params: Input module parameters
+            :return: First, it returns list of result dataframe.
+            Second, it returns list of execution times.
+        """
+        result, execution_times = zip(*map(lambda task: measure_speed(
+            task[0], project_dir=project_dir, previous_result=previous_result, **task[1]),
+                                           zip(input_modules, input_module_params)))
+        average_times = list(map(lambda x: x / len(result[0]), execution_times))
+
+        # run metrics before filtering
+        if strategies.get('metrics') is None:
+            raise ValueError("You must at least one metrics for retrieval evaluation.")
+        result = list(map(lambda x: evaluate_retrieval_node(x, retrieval_gt, strategies.get('metrics'),
+                                                            qa_df['query'].tolist(),
+                                                            qa_df['generation_gt'].tolist()), result))
+
+        return result, average_times
+
+    def save_and_summary(input_modules, input_module_params, result_list,
+                         execution_time_list, filename_start: int):
+        """
+            Save the result and make summary file
+
+            :param input_modules: Input modules
+            :param input_module_params: Input module parameters
+            :param result_list: Result list
+            :param execution_time_list: Execution times
+            :param filename_start: The first filename to use
+            :return: First, it returns list of result dataframe.
+            Second, it returns list of execution times.
+        """
+
+        # save results to folder
+        filepaths = list(map(lambda x: os.path.join(save_dir, f'{x}.parquet'),
+                             range(filename_start, filename_start + len(input_modules))))
+        list(map(lambda x: x[0].to_parquet(x[1], index=False), zip(result_list, filepaths)))  # execute save to parquet
+        filename_list = list(map(lambda x: os.path.basename(x), filepaths))
+
+        summary_df = pd.DataFrame({
+            'filename': filename_list,
+            'module_name': list(map(lambda module: module.__name__, input_modules)),
+            'module_params': input_module_params,
+            'execution_time': execution_time_list,
+            **{metric: list(map(lambda result: result[metric].mean(), result_list)) for metric in
+               strategies.get('metrics')},
+        })
+        summary_df.to_csv(os.path.join(save_dir, 'summary.csv'), index=False)
+        return summary_df
+
+    def find_best(results, average_times, filenames):
+        # filter by strategies
+        if strategies.get('speed_threshold') is not None:
+            results, filenames = filter_by_threshold(results, average_times, strategies['speed_threshold'], filenames)
+        selected_result, selected_filename = select_best(results, strategies.get('metrics'), filenames,
+                                                         strategies.get('strategy', 'mean'))
+        return selected_result, selected_filename
+
+    filename_first = 0
+    # run semantic modules
+    logger.info(f"Running retrieval node - semantic retrieval module...")
+    if any([module.__name__ in semantic_module_names for module in modules]):
+        semantic_modules, semantic_module_params = zip(*filter(lambda x: x[0].__name__ in semantic_module_names,
+                                                               zip(modules, module_params)))
+        semantic_results, semantic_times = run(semantic_modules, semantic_module_params)
+        semantic_summary_df = save_and_summary(semantic_modules, semantic_module_params,
+                                               semantic_results, semantic_times, filename_first)
+        semantic_selected_result, semantic_selected_filename = find_best(semantic_results, semantic_times,
+                                                                         semantic_summary_df['filename'].tolist())
+        semantic_summary_df['is_best'] = semantic_summary_df['filename'] == semantic_selected_filename
+        filename_first += len(semantic_modules)
+    else:
+        semantic_selected_filename, semantic_summary_df, semantic_results, semantic_times = None, pd.DataFrame(), [], []
+    # run lexical modules
+    logger.info(f"Running retrieval node - lexical retrieval module...")
+    if any([module.__name__ in lexical_module_names for module in modules]):
+        lexical_modules, lexical_module_params = zip(*filter(lambda x: x[0].__name__ in lexical_module_names,
+                                                             zip(modules, module_params)))
+        lexical_results, lexical_times = run(lexical_modules, lexical_module_params)
+        lexical_summary_df = save_and_summary(lexical_modules, lexical_module_params,
+                                              lexical_results, lexical_times, filename_first)
+        lexical_selected_result, lexical_selected_filename = find_best(lexical_results, lexical_times,
+                                                                       lexical_summary_df['filename'].tolist())
+        lexical_summary_df['is_best'] = lexical_summary_df['filename'] == lexical_selected_filename
+        filename_first += len(lexical_modules)
+    else:
+        lexical_selected_filename, lexical_summary_df, lexical_results, lexical_times = None, pd.DataFrame(), [], []
+
+    logger.info(f"Running retrieval node - hybrid retrieval module...")
+    # Next, run hybrid retrieval
+    if any([module.__name__ in hybrid_module_names for module in modules]):
+        hybrid_modules, hybrid_module_params = zip(*filter(lambda x: x[0].__name__ in hybrid_module_names,
+                                                           zip(modules, module_params)))
+        if all(['target_module_params' in x for x in hybrid_module_params]):  # for Runner.run
+            # If target_module_params are already given, run hybrid retrieval directly
+            hybrid_results, hybrid_times = run(hybrid_modules, hybrid_module_params)
+            hybrid_summary_df = save_and_summary(hybrid_modules, hybrid_module_params,
+                                                 hybrid_results, hybrid_times, filename_first)
+            filename_first += len(hybrid_modules)
+        else:  # for Evaluator
+            # get id and score
+            ids_scores = get_ids_and_scores(save_dir, [semantic_selected_filename, lexical_selected_filename])
+            hybrid_module_params = list(map(lambda x: {**x, **ids_scores}, hybrid_module_params))
+
+            # optimize each modules
+            real_hybrid_times = [get_hybrid_execution_times(semantic_summary_df, lexical_summary_df)
+                                 ] * len(hybrid_module_params)
+            hybrid_times = real_hybrid_times.copy()
+            hybrid_results = []
+            for module, module_param in zip(hybrid_modules, hybrid_module_params):
+                module_result_df, module_best_weight = optimize_hybrid(module, module_param, strategies,
+                                                                       retrieval_gt, qa_df,
+                                                                       project_dir, previous_result)
+                module_param['weight'] = module_best_weight
+                hybrid_results.append(module_result_df)
+
+            hybrid_summary_df = save_and_summary(hybrid_modules, hybrid_module_params,
+                                                 hybrid_results, hybrid_times, filename_first)
+            filename_first += len(hybrid_modules)
+            hybrid_summary_df['execution_time'] = hybrid_times
+            best_semantic_summary_row = semantic_summary_df.loc[semantic_summary_df['is_best'] == True].iloc[0]
+            best_lexical_summary_row = lexical_summary_df.loc[lexical_summary_df['is_best'] == True].iloc[0]
+            target_modules = (best_semantic_summary_row['module_name'], best_lexical_summary_row['module_name'])
+            target_module_params = (
+                best_semantic_summary_row['module_params'], best_lexical_summary_row['module_params'])
+            hybrid_summary_df = edit_summary_df_params(hybrid_summary_df, target_modules, target_module_params)
+    else:
+        if any([module.__name__ in hybrid_module_names for module in modules]):
+            logger.warning("You must at least one semantic module and lexical module for hybrid evaluation."
+                           "Passing hybrid module.")
+        hybrid_selected_filename, hybrid_summary_df, hybrid_results, hybrid_times = None, pd.DataFrame(), [], []
+
+    summary = pd.concat([semantic_summary_df, lexical_summary_df, hybrid_summary_df], ignore_index=True)
+    results = semantic_results + lexical_results + hybrid_results
+    average_times = semantic_times + lexical_times + hybrid_times
+    filenames = summary['filename'].tolist()
+
+    # filter by strategies
+    selected_result, selected_filename = find_best(results, average_times, filenames)
+    best_result = pd.concat([previous_result, selected_result], axis=1)
+
+    # add summary.csv 'is_best' column
+    summary['is_best'] = summary['filename'] == selected_filename
+
+    # save the result files
+    best_result.to_parquet(os.path.join(save_dir, f'best_{os.path.splitext(selected_filename)[0]}.parquet'),
+                           index=False)
+    summary.to_csv(os.path.join(save_dir, 'summary.csv'), index=False)
+    return best_result
+
+
+def evaluate_retrieval_node(result_df: pd.DataFrame, retrieval_gt, metrics,
+                            queries: List[str], generation_gt: List[List[str]]) -> pd.DataFrame:
+    """
+    Evaluate retrieval node from retrieval node result dataframe.
+
+    :param result_df: The result dataframe from a retrieval node.
+    :param retrieval_gt: Ground truth for retrieval from qa dataset.
+    :param metrics: Metric list from input strategies.
+    :param queries: Query list from input strategies.
+    :param generation_gt: Ground truth for generation from qa dataset.
+    :return: Return result_df with metrics columns.
+        The columns will be 'retrieved_contents', 'retrieved_ids', 'retrieve_scores', and metric names.
+    """
+
+    @evaluate_retrieval(retrieval_gt=retrieval_gt, metrics=metrics, queries=queries, generation_gt=generation_gt)
+    def evaluate_this_module(df: pd.DataFrame):
+        return df['retrieved_contents'].tolist(), df['retrieved_ids'].tolist(), df['retrieve_scores'].tolist()
+
+    return evaluate_this_module(result_df)
+
+
+def edit_summary_df_params(summary_df: pd.DataFrame, target_modules, target_module_params) -> pd.DataFrame:
+    def delete_ids_scores(x):
+        del x['ids']
+        del x['scores']
+        return x
+
+    summary_df['module_params'] = summary_df['module_params'].apply(delete_ids_scores)
+    summary_df['new_params'] = [{'target_modules': target_modules,
+                                 'target_module_params': target_module_params}] * len(summary_df)
+    summary_df['module_params'] = summary_df.apply(lambda row: {**row['module_params'], **row['new_params']}, axis=1)
+    summary_df = summary_df.drop(columns=['new_params'])
+    return summary_df
+
+
+def get_ids_and_scores(node_dir: str, filenames: List[str]) -> Dict:
+    best_results_df = list(
+        map(lambda filename: pd.read_parquet(os.path.join(node_dir, filename), engine='pyarrow'), filenames))
+    ids = tuple(map(lambda df: df['retrieved_ids'].apply(list).tolist(), best_results_df))
+    scores = tuple(map(lambda df: df['retrieve_scores'].apply(list).tolist(), best_results_df))
+    return {
+        'ids': ids,
+        'scores': scores,
+    }
+
+
+def get_hybrid_execution_times(lexical_summary, semantic_summary) -> float:
+    lexical_execution_time = lexical_summary.loc[lexical_summary['is_best'] == True].iloc[0]['execution_time']
+    semantic_execution_time = semantic_summary.loc[semantic_summary['is_best'] == True].iloc[0]['execution_time']
+    return lexical_execution_time + semantic_execution_time
+
+
+def optimize_hybrid(hybrid_module_func: Callable, hybrid_module_param: Dict,
+                    strategy: Dict, retrieval_gt, qa_df: pd.DataFrame,
+                    project_dir, previous_result):
+    if hybrid_module_func.__name__ == 'hybrid_rrf':
+        weight_range = hybrid_module_param.pop('weight_range', (4, 80))
+        test_weight_size = weight_range[1] - weight_range[0] + 1
+    else:
+        weight_range = hybrid_module_param.pop('weight_range', (0.0, 1.0))
+        test_weight_size = hybrid_module_param.pop('test_weight_size', 101)
+
+    weight_candidates = np.linspace(weight_range[0], weight_range[1], test_weight_size).tolist()
+
+    result_list = []
+    for weight_value in tqdm(weight_candidates):
+        result_df = hybrid_module_func(project_dir=project_dir, previous_result=previous_result,
+                                       weight=weight_value, **hybrid_module_param)
+        result_list.append(result_df)
+
+        # evaluate here
+    if strategy.get('metrics') is None:
+        raise ValueError("You must at least one metrics for retrieval evaluation.")
+    result_list = list(map(lambda x: evaluate_retrieval_node(x, retrieval_gt, strategy.get('metrics'),
+                                                             qa_df['query'].tolist(),
+                                                             qa_df['generation_gt'].tolist()), result_list))
+
+    # select best result
+    best_result_df, best_weight = select_best(result_list, strategy.get('metrics'), metadatas=weight_candidates,
+                                              strategy_name=strategy.get('strategy', 'normalize_mean'))
+    return best_result_df, best_weight

{autorag-0.2.9 → autorag-0.2.10}/autorag/support.py

@@ -24,8 +24,6 @@ def get_support_modules(module_name: str) -> Callable:
         'vectordb': ('autorag.nodes.retrieval', 'vectordb'),
         'hybrid_rrf': ('autorag.nodes.retrieval', 'hybrid_rrf'),
         'hybrid_cc': ('autorag.nodes.retrieval', 'hybrid_cc'),
-        'hybrid_rsf': ('autorag.nodes.retrieval', 'hybrid_rsf'),
-        'hybrid_dbsf': ('autorag.nodes.retrieval', 'hybrid_dbsf'),
         # passage_augmenter
         'prev_next_augmenter': ('autorag.nodes.passageaugmenter', 'prev_next_augmenter'),
         'pass_passage_augmenter': ('autorag.nodes.passageaugmenter', 'pass_passage_augmenter'),

{autorag-0.2.9 → autorag-0.2.10}/docs/source/api_spec/autorag.nodes.retrieval.rst

@@ -28,14 +28,6 @@ autorag.nodes.retrieval.hybrid\_cc module
    :undoc-members:
    :show-inheritance:
 
-autorag.nodes.retrieval.hybrid\_dbsf module
--------------------------------------------
-
-.. automodule:: autorag.nodes.retrieval.hybrid_dbsf
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 autorag.nodes.retrieval.hybrid\_rrf module
 ------------------------------------------
 
@@ -44,14 +36,6 @@ autorag.nodes.retrieval.hybrid\_rrf module
    :undoc-members:
    :show-inheritance:
 
-autorag.nodes.retrieval.hybrid\_rsf module
-------------------------------------------
-
-.. automodule:: autorag.nodes.retrieval.hybrid_rsf
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 autorag.nodes.retrieval.run module
 ----------------------------------
 

{autorag-0.2.9 → autorag-0.2.10}/docs/source/evaluate_metrics/generation.md

@@ -63,7 +63,7 @@ at [here](https://medium.com/@autorag/sem-score-maybe-the-answer-to-rag-evaluati
 
 ## 5. G-Eval
 
-### 📌Definition
+### 📌 Definition
 
 Here is the [link](https://arxiv.org/abs/2303.16634) that introduced ***G-Eval***
 
@@ -77,14 +77,14 @@ So, in AutoRAG, we use **G-Eval with GPT-4**
 
 ---
 
-### 🍀1. Coherence
+### 5-1. Coherence
 
 - Evaluate whether the answer is logically consistent and flows naturally.
 - Evaluate the connections between sentences and how they fit into the overall context.
 
 ---
 
-### 🍀2. Consistency
+### 5-2. Consistency
 
 - Evaluate whether the answer is consistent with and does not contradict the question asked or the information
   presented.
@@ -93,17 +93,28 @@ So, in AutoRAG, we use **G-Eval with GPT-4**
 
 ---
 
-### 🍀3. Fluency
+### 5-3. Fluency
 
 - Evaluate answers for fluency
 
 ---
 
-### 🍀4. Relevance
+### 5-4. Relevance
 
 - Evaluate how well the answer meets the question's requirements
 - A highly relevant answer should be directly related to the question's core topic or keyword.
 
+### ❗How to use specific G-Eval metrics
+
+You can use specific G-Eval metrics to use `metrics` parameter.
+
+Here is an example yaml file that uses **G-Eval consistency** metric.
+
+```yaml
+- metric_name: g_eval
+  metrics: [ consistency ]
+```
+
 ## 6. Bert Score
 
 ### 📌Definition