AutoRAG 0.2.8__tar.gz → 0.2.10__tar.gz

{autorag-0.2.8 → autorag-0.2.10}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.8
+Version: 0.2.10
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

{autorag-0.2.8 → autorag-0.2.10}/AutoRAG.egg-info/SOURCES.txt

@@ -104,6 +104,7 @@ autorag/nodes/promptmaker/base.py
 autorag/nodes/promptmaker/fstring.py
 autorag/nodes/promptmaker/long_context_reorder.py
 autorag/nodes/promptmaker/run.py
+autorag/nodes/promptmaker/window_replacement.py
 autorag/nodes/queryexpansion/__init__.py
 autorag/nodes/queryexpansion/base.py
 autorag/nodes/queryexpansion/hyde.py
@@ -115,9 +116,7 @@ autorag/nodes/retrieval/__init__.py
 autorag/nodes/retrieval/base.py
 autorag/nodes/retrieval/bm25.py
 autorag/nodes/retrieval/hybrid_cc.py
-autorag/nodes/retrieval/hybrid_dbsf.py
 autorag/nodes/retrieval/hybrid_rrf.py
-autorag/nodes/retrieval/hybrid_rsf.py
 autorag/nodes/retrieval/run.py
 autorag/nodes/retrieval/vectordb.py
 autorag/schema/__init__.py
@@ -227,21 +226,19 @@ docs/source/nodes/passage_reranker/upr.md
 docs/source/nodes/prompt_maker/fstring.md
 docs/source/nodes/prompt_maker/long_context_reorder.md
 docs/source/nodes/prompt_maker/prompt_maker.md
+docs/source/nodes/prompt_maker/window_replacement.md
 docs/source/nodes/query_expansion/hyde.md
 docs/source/nodes/query_expansion/multi_query_expansion.md
 docs/source/nodes/query_expansion/query_decompose.md
 docs/source/nodes/query_expansion/query_expansion.md
 docs/source/nodes/retrieval/bm25.md
 docs/source/nodes/retrieval/hybrid_cc.md
-docs/source/nodes/retrieval/hybrid_dbsf.md
 docs/source/nodes/retrieval/hybrid_rrf.md
-docs/source/nodes/retrieval/hybrid_rsf.md
 docs/source/nodes/retrieval/retrieval.md
 docs/source/nodes/retrieval/vectordb.md
 docs/source/optimization/custom_config.md
 docs/source/optimization/folder_structure.md
 docs/source/optimization/optimization.md
-docs/source/optimization/sample_full_config.yaml
 docs/source/optimization/strategies.md
 docs/source/roadmap/modular_rag.md
 sample_config/compact_local.yaml
@@ -250,6 +247,7 @@ sample_config/config_korean.yaml
 sample_config/extracted_sample.yaml
 sample_config/full.yaml
 sample_config/simple_local.yaml
+sample_config/simple_ollama.yaml
 sample_config/simple_openai.yaml
 sample_dataset/README.md
 sample_dataset/eli5/load_eli5_dataset.py
@@ -323,6 +321,7 @@ tests/autorag/nodes/promptmaker/test_fstring.py
 tests/autorag/nodes/promptmaker/test_long_context_reorder.py
 tests/autorag/nodes/promptmaker/test_prompt_maker_base.py
 tests/autorag/nodes/promptmaker/test_prompt_maker_run.py
+tests/autorag/nodes/promptmaker/test_window_replacement.py
 tests/autorag/nodes/queryexpansion/test_hyde.py
 tests/autorag/nodes/queryexpansion/test_multi_query_expansion.py
 tests/autorag/nodes/queryexpansion/test_pass_query_expansion.py
@@ -332,9 +331,7 @@ tests/autorag/nodes/queryexpansion/test_query_expansion_run.py
 tests/autorag/nodes/retrieval/test_bm25.py
 tests/autorag/nodes/retrieval/test_hybrid_base.py
 tests/autorag/nodes/retrieval/test_hybrid_cc.py
-tests/autorag/nodes/retrieval/test_hybrid_dbsf.py
 tests/autorag/nodes/retrieval/test_hybrid_rrf.py
-tests/autorag/nodes/retrieval/test_hybrid_rsf.py
 tests/autorag/nodes/retrieval/test_retrieval_base.py
 tests/autorag/nodes/retrieval/test_run_retrieval_node.py
 tests/autorag/nodes/retrieval/test_vectordb.py

{autorag-0.2.8 → autorag-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.8
+Version: 0.2.10
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

autorag-0.2.10/autorag/VERSION

@@ -0,0 +1 @@
+0.2.10

{autorag-0.2.8 → autorag-0.2.10}/autorag/data/qacreation/base.py

@@ -3,6 +3,7 @@ import uuid
 from typing import Callable, Optional
 
 import pandas as pd
+from tqdm import tqdm
 
 from autorag.utils.util import save_parquet_safe
 
@@ -15,6 +16,7 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
                            output_filepath: Optional[str] = None,
                            upsert: bool = False,
                            random_state: int = 42,
+                           cache_batch: int = 32,
                            **kwargs) -> pd.DataFrame:
     """
     Make single content (single-hop, single-document) QA dataset using given qa_creation_func.
@@ -32,6 +34,9 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
     :param upsert: If true, the function will overwrite the existing file if it exists.
         Default is False.
     :param random_state: The random state for sampling corpus from the given corpus_df.
+    :param cache_batch: The number of batches to use for caching the generated QA dataset.
+        When the cache_batch size data is generated, the dataset will save to the designated output_filepath.
+        If the cache_batch size is too small, the process time will be longer.
     :param kwargs: The keyword arguments for qa_creation_func.
     :return: QA dataset dataframe.
         You can save this as parquet file to use at AutoRAG.
@@ -44,24 +49,31 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
     sampled_corpus = corpus_df.sample(n=content_size, random_state=random_state)
     sampled_corpus = sampled_corpus.reset_index(drop=True)
 
-    qa = qa_creation_func(contents=sampled_corpus['contents'].tolist(), **kwargs)
-    qa_data = pd.DataFrame({
-        'qid': [str(uuid.uuid4()) for _ in range(len(qa))],
-        'qa': qa,
-        'retrieval_gt': sampled_corpus['doc_id'].tolist(),
-    })
-    qa_data = qa_data.explode('qa', ignore_index=True)
-
     def make_query_generation_gt(row):
         return row['qa']['query'], row['qa']['generation_gt']
 
-    qa_data[['query', 'generation_gt']] = qa_data.apply(make_query_generation_gt, axis=1, result_type='expand')
-    qa_data = qa_data.drop(columns=['qa'])
+    qa_data = pd.DataFrame()
+    for idx, i in tqdm(enumerate(range(0, len(sampled_corpus), cache_batch))):
+        qa = qa_creation_func(contents=sampled_corpus['contents'].tolist()[i:i + cache_batch], **kwargs)
+
+        temp_qa_data = pd.DataFrame({
+            'qa': qa,
+            'retrieval_gt': sampled_corpus['doc_id'].tolist()[i:i + cache_batch],
+        })
+        temp_qa_data = temp_qa_data.explode('qa', ignore_index=True)
+        temp_qa_data['qid'] = [str(uuid.uuid4()) for _ in range(len(temp_qa_data))]
+        temp_qa_data[['query', 'generation_gt']] = temp_qa_data.apply(make_query_generation_gt, axis=1,
+                                                                      result_type='expand')
+        temp_qa_data = temp_qa_data.drop(columns=['qa'])
 
-    qa_data['retrieval_gt'] = qa_data['retrieval_gt'].apply(lambda x: [[x]])
-    qa_data['generation_gt'] = qa_data['generation_gt'].apply(lambda x: [x])
+        temp_qa_data['retrieval_gt'] = temp_qa_data['retrieval_gt'].apply(lambda x: [[x]])
+        temp_qa_data['generation_gt'] = temp_qa_data['generation_gt'].apply(lambda x: [x])
 
-    if output_filepath is not None:
-        save_parquet_safe(qa_data, output_filepath, upsert=upsert)
+        if idx == 0:
+            qa_data = temp_qa_data
+        else:
+            qa_data = pd.concat([qa_data, temp_qa_data], ignore_index=True)
+        if output_filepath is not None:
+            save_parquet_safe(qa_data, output_filepath, upsert=upsert)
 
     return qa_data

{autorag-0.2.8 → autorag-0.2.10}/autorag/evaluation/metric/generation.py

@@ -6,7 +6,7 @@ from typing import List, Optional
 
 import evaluate
 import pandas as pd
-import sacrebleu
+from sacrebleu.metrics.bleu import BLEU
 import torch
 from llama_index.core.embeddings import BaseEmbedding
 from llama_index.embeddings.openai import OpenAIEmbedding
@@ -64,12 +64,25 @@ def huggingface_evaluate(instance, key: str,
     return result
 
 
-@generation_metric
-def bleu(gt: List[str], pred: str, **kwargs) -> float:
+@convert_inputs_to_list
+def bleu(generation_gt: List[List[str]], generations: [str], tokenize: str|None = None, smooth_method: str = 'exp', smooth_value: Optional[float] = None, max_ngram_order: int = 4, trg_lang: str = '', **kwargs) -> List[float]:
     """
-    Compute bleu score for generation.
+    Computes the BLEU metric given pred and ground-truth.
+
+    :param tokenize: The tokenizer to use. If None, defaults to language-specific tokenizers with '13a' as the fallback default. check #https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/metrics/bleu.py
+    :param smooth_method: The smoothing method to use ('floor', 'add-k', 'exp' or 'none').
+    :param smooth_value: The smoothing value for `floor` and `add-k` methods. `None` falls back to default value.
+    :param max_ngram_order: If given, it overrides the maximum n-gram order (default: 4) when computing precisions.
+    :param trg_lang: An optional language code to raise potential tokenizer warnings.
+    :param generation_gt: A list of ground truth.
+        Must be 2-d list of string.
+        Because it can be a multiple ground truth.
+    :param generations: A list of generations that LLM generated.
     """
-    return sacrebleu.sentence_bleu(pred, gt, **kwargs).score
+    bleu = BLEU(tokenize=tokenize, smooth_method=smooth_method, smooth_value=smooth_value, max_ngram_order=max_ngram_order, trg_lang=trg_lang, **kwargs)
+
+    result = list(map(lambda x: bleu.sentence_score(x[0], x[1]).score, zip(generations, generation_gt)))
+    return result
 
 
 @convert_inputs_to_list

{autorag-0.2.8 → autorag-0.2.10}/autorag/nodes/promptmaker/__init__.py

@@ -1,2 +1,3 @@
 from .fstring import fstring
 from .long_context_reorder import long_context_reorder
+from .window_replacement import window_replacement

{autorag-0.2.8 → autorag-0.2.10}/autorag/nodes/promptmaker/base.py

@@ -1,11 +1,12 @@
 import functools
 import logging
+import os
 from pathlib import Path
 from typing import List, Union
 
 import pandas as pd
 
-from autorag.utils import result_to_dataframe
+from autorag.utils import result_to_dataframe, fetch_contents
 
 logger = logging.getLogger("AutoRAG")
 
@@ -31,6 +32,14 @@ def prompt_maker_node(func):
             assert "retrieve_scores" in previous_result.columns, "previous_result must have retrieve_scores column."
             retrieve_scores = previous_result["retrieve_scores"].tolist()
             return func(prompt, query, retrieved_contents, retrieve_scores)
+        elif func.__name__ == 'window_replacement':
+            retrieved_ids = previous_result["retrieved_ids"].tolist()
+            # load corpus
+            data_dir = os.path.join(project_dir, "data")
+            corpus_data = pd.read_parquet(os.path.join(data_dir, "corpus.parquet"), engine='pyarrow')
+            # get metadata from corpus
+            retrieved_metadata = fetch_contents(corpus_data, retrieved_ids, column_name='metadata')
+            return func(prompt, query, retrieved_contents, retrieved_metadata)
         else:
             raise NotImplementedError(f"Module {func.__name__} is not implemented or not supported.")
 

autorag-0.2.10/autorag/nodes/promptmaker/window_replacement.py

@@ -0,0 +1,46 @@
+import logging
+from typing import List, Dict
+
+from autorag.nodes.promptmaker.base import prompt_maker_node
+
+logger = logging.getLogger("AutoRAG")
+
+
+@prompt_maker_node
+def window_replacement(prompt: str, queries: List[str],
+                       retrieved_contents: List[List[str]],
+                       retrieved_metadata: List[List[Dict]]) -> List[str]:
+    """
+    Replace retrieved_contents with window to create a Prompt
+    (only available for corpus chunked with Sentence window method)
+    You must type a prompt or prompt list at config yaml file like this:
+
+    .. Code:: yaml
+    nodes:
+    - node_type: prompt_maker
+      modules:
+      - module_type: window_replacement
+        prompt: [Answer this question: {query} \n\n {retrieved_contents},
+        Read the passages carefully and answer this question: {query} \n\n Passages: {retrieved_contents}]
+
+    :param prompt: A prompt string.
+    :param queries: List of query strings.
+    :param retrieved_contents: List of retrieved contents.
+    :param retrieved_metadata: List of retrieved metadata.
+    :return: Prompts that made by window_replacement.
+    """
+
+    def window_replacement_row(_prompt: str, _query: str, _retrieved_contents, _retrieved_metadata: List[Dict]) -> str:
+        window_list = []
+        for content, metadata in zip(_retrieved_contents, _retrieved_metadata):
+            if 'window' in metadata:
+                window_list.append(metadata['window'])
+            else:
+                window_list.append(content)
+                logger.info("Only available for corpus chunked with Sentence window method."
+                            "window_replacement will not proceed.")
+        contents_str = "\n\n".join(window_list)
+        return _prompt.format(query=_query, retrieved_contents=contents_str)
+
+    return list(map(lambda x: window_replacement_row(prompt, x[0], x[1], x[2]),
+                    zip(queries, retrieved_contents, retrieved_metadata)))

{autorag-0.2.8 → autorag-0.2.10}/autorag/nodes/retrieval/__init__.py

@@ -1,7 +1,5 @@
 from .base import retrieval_node
 from .bm25 import bm25
 from .hybrid_cc import hybrid_cc
-from .hybrid_dbsf import hybrid_dbsf
 from .hybrid_rrf import hybrid_rrf
-from .hybrid_rsf import hybrid_rsf
 from .vectordb import vectordb

{autorag-0.2.8 → autorag-0.2.10}/autorag/nodes/retrieval/base.py

@@ -32,7 +32,6 @@ def retrieval_node(func):
             project_dir: Union[str, Path],
             previous_result: pd.DataFrame,
             **kwargs) -> Tuple[List[List[str]], List[List[str]], List[List[float]]]:
-        logger.info(f"Running retrieval node - {func.__name__} module...")
         validate_qa_dataset(previous_result)
         resources_dir = os.path.join(project_dir, "resources")
         data_dir = os.path.join(project_dir, "data")
@@ -75,10 +74,10 @@ def retrieval_node(func):
             del embedding_model
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
-        elif func.__name__ in ["hybrid_rrf", "hybrid_cc", "hybrid_rsf", "hybrid_dbsf"]:
-            if 'ids' in kwargs and 'scores' in kwargs:
+        elif func.__name__ in ["hybrid_rrf", "hybrid_cc"]:
+            if 'ids' in kwargs and 'scores' in kwargs:  # ordinary run_evaluate
                 ids, scores = func(**kwargs)
-            else:
+            else:  # => for Runner.run
                 if not ('target_modules' in kwargs and 'target_module_params' in kwargs):
                     raise ValueError(
                         f"If there are no ids and scores specified, target_modules and target_module_params must be specified for using {func.__name__}.")

autorag-0.2.10/autorag/nodes/retrieval/hybrid_cc.py

@@ -0,0 +1,137 @@
+from typing import Tuple, List
+
+import numpy as np
+import pandas as pd
+
+from autorag.nodes.retrieval import retrieval_node
+
+
+def normalize_mm(scores: List[str], fixed_min_value: float = 0):
+    arr = np.array(scores)
+    max_value = np.max(arr)
+    min_value = np.min(arr)
+    norm_score = (arr - min_value) / (max_value - min_value)
+    return norm_score
+
+
+def normalize_tmm(scores: List[str], fixed_min_value: float):
+    arr = np.array(scores)
+    max_value = np.max(arr)
+    norm_score = (arr - fixed_min_value) / (max_value - fixed_min_value)
+    return norm_score
+
+
+def normalize_z(scores: List[str], fixed_min_value: float = 0):
+    arr = np.array(scores)
+    mean_value = np.mean(arr)
+    std_value = np.std(arr)
+    norm_score = (arr - mean_value) / std_value
+    return norm_score
+
+
+def normalize_dbsf(scores: List[str], fixed_min_value: float = 0):
+    arr = np.array(scores)
+    mean_value = np.mean(arr)
+    std_value = np.std(arr)
+    min_value = mean_value - 3 * std_value
+    max_value = mean_value + 3 * std_value
+    norm_score = (arr - min_value) / (max_value - min_value)
+    return norm_score
+
+
+normalize_method_dict = {
+    'mm': normalize_mm,
+    'tmm': normalize_tmm,
+    'z': normalize_z,
+    'dbsf': normalize_dbsf,
+}
+
+
+@retrieval_node
+def hybrid_cc(
+        ids: Tuple,
+        scores: Tuple,
+        top_k: int,
+        weight: float,
+        normalize_method: str = 'mm',
+        semantic_theoretical_min_value: float = -1.0,
+        lexical_theoretical_min_value: float = 0.0,
+) -> Tuple[List[List[str]], List[List[float]]]:
+    """
+    Hybrid CC function.
+    CC (convex combination) is a method to fuse lexical and semantic retrieval results.
+    It is a method that first normalizes the scores of each retrieval result,
+    and then combines them with the given weights.
+    It is uniquer than other retrieval modules, because it does not really execute retrieval,
+    but just fuse the results of other retrieval functions.
+    So you have to run more than two retrieval modules before running this function.
+    And collect ids and scores result from each retrieval module.
+    Make it as tuple and input it to this function.
+
+    :param ids: The tuple of ids that you want to fuse.
+        The length of this must be the same as the length of scores.
+        The semantic retrieval ids must be the first index.
+    :param scores: The retrieve scores that you want to fuse.
+        The length of this must be the same as the length of ids.
+        The semantic retrieval scores must be the first index.
+    :param top_k: The number of passages to be retrieved.
+    :param normalize_method: The normalization method to use.
+      There are some normalization method that you can use at the hybrid cc method.
+      AutoRAG support following.
+        - `mm`: Min-max scaling
+        - `tmm`: Theoretical min-max scaling
+        - `z`: z-score normalization
+        - `dbsf`: 3-sigma normalization
+    :param weight: The weight value. If the weight is 1.0, it means the
+      weight to the semantic module will be 1.0 and weight to the lexical module will be 0.0.
+    :param semantic_theoretical_min_value: This value used by `tmm` normalization method. You can set the
+        theoretical minimum value by yourself. Default is -1.
+    :param lexical_theoretical_min_value: This value used by `tmm` normalization method. You can set the
+        theoretical minimum value by yourself. Default is 0.
+    :return: The tuple of ids and fused scores that fused by CC. Plus, the third element is selected weight value.
+    """
+    assert len(ids) == len(scores), "The length of ids and scores must be the same."
+    assert len(ids) > 1, "You must input more than one retrieval results."
+    assert top_k > 0, "top_k must be greater than 0."
+    assert weight >= 0, "The weight must be greater than 0."
+    assert weight <= 1, "The weight must be less than 1."
+
+    df = pd.DataFrame({
+        'semantic_ids': ids[0],
+        'lexical_ids': ids[1],
+        'semantic_score': scores[0],
+        'lexical_score': scores[1],
+    })
+
+    def cc_pure_apply(row):
+        return fuse_per_query(row['semantic_ids'], row['lexical_ids'],
+                              row['semantic_score'], row['lexical_score'],
+                              normalize_method=normalize_method,
+                              weight=weight, top_k=top_k,
+                              semantic_theoretical_min_value=semantic_theoretical_min_value,
+                              lexical_theoretical_min_value=lexical_theoretical_min_value)
+
+    # fixed weight
+    df[['cc_id', 'cc_score']] = df.apply(lambda row: cc_pure_apply(row), axis=1,
+                                         result_type='expand')
+    return df['cc_id'].tolist(), df['cc_score'].tolist()
+
+
+def fuse_per_query(semantic_ids: List[str], lexical_ids: List[str],
+                   semantic_scores: List[float], lexical_scores: List[float],
+                   normalize_method: str,
+                   weight: float,
+                   top_k: int,
+                   semantic_theoretical_min_value: float,
+                   lexical_theoretical_min_value: float):
+    normalize_func = normalize_method_dict[normalize_method]
+    norm_semantic_scores = normalize_func(semantic_scores, semantic_theoretical_min_value)
+    norm_lexical_scores = normalize_func(lexical_scores, lexical_theoretical_min_value)
+    ids = [semantic_ids, lexical_ids]
+    scores = [norm_semantic_scores, norm_lexical_scores]
+    df = pd.concat([pd.Series(dict(zip(_id, score))) for _id, score in zip(ids, scores)], axis=1)
+    df.columns = ['semantic', 'lexical']
+    df = df.fillna(0)
+    df['weighted_sum'] = df.mul((weight, 1.0 - weight)).sum(axis=1)
+    df = df.sort_values(by='weighted_sum', ascending=False)
+    return df.index.tolist()[:top_k], df['weighted_sum'][:top_k].tolist()

{autorag-0.2.8 → autorag-0.2.10}/autorag/nodes/retrieval/hybrid_rrf.py

@@ -10,7 +10,8 @@ def hybrid_rrf(
         ids: Tuple,
         scores: Tuple,
         top_k: int,
-        rrf_k: int = 60) -> Tuple[List[List[str]], List[List[float]]]:
+        weight: int = 60,
+        rrf_k: int = -1, ) -> Tuple[List[List[str]], List[List[float]]]:
     """
     Hybrid RRF function.
     RRF (Rank Reciprocal Fusion) is a method to fuse multiple retrieval results.
@@ -27,15 +28,23 @@ def hybrid_rrf(
     :param scores: The retrieve scores that you want to fuse.
         The length of this must be the same as the length of ids.
     :param top_k: The number of passages to be retrieved.
-    :param rrf_k: Hyperparameter for RRF.
+    :param weight: Hyperparameter for RRF.
+        It was originally rrf_k value.
         Default is 60.
         For more information, please visit our documentation.
+    :param rrf_k: (Deprecated) Hyperparameter for RRF.
+        It was originally rrf_k value. Will remove at further version.
     :return: The tuple of ids and fused scores that fused by RRF.
     """
     assert len(ids) == len(scores), "The length of ids and scores must be the same."
     assert len(ids) > 1, "You must input more than one retrieval results."
     assert top_k > 0, "top_k must be greater than 0."
-    assert rrf_k > 0, "rrf_k must be greater than 0."
+    assert weight > 0, "rrf_k must be greater than 0."
+
+    if rrf_k != -1:
+        weight = int(rrf_k)
+    else:
+        weight = int(weight)
 
     id_df = pd.DataFrame({f'id_{i}': id_list for i, id_list in enumerate(ids)})
     score_df = pd.DataFrame({f'score_{i}': score_list for i, score_list in enumerate(scores)})
@@ -44,9 +53,9 @@ def hybrid_rrf(
     def rrf_pure_apply(row):
         ids_tuple = tuple(row[[f'id_{i}' for i in range(len(ids))]].values)
         scores_tuple = tuple(row[[f'score_{i}' for i in range(len(scores))]].values)
-        return pd.Series(rrf_pure(ids_tuple, scores_tuple, rrf_k, top_k))
+        return pd.Series(rrf_pure(ids_tuple, scores_tuple, weight, top_k))
 
-    df[['rrf_id', 'rrf_score']] = df.swifter.apply(rrf_pure_apply, axis=1)
+    df[['rrf_id', 'rrf_score']] = df.apply(rrf_pure_apply, axis=1)
     return df['rrf_id'].tolist(), df['rrf_score'].tolist()