AutoRAG 0.2.8__tar.gz → 0.2.9__tar.gz

{autorag-0.2.8 → autorag-0.2.9}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.8
+Version: 0.2.9
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

{autorag-0.2.8 → autorag-0.2.9}/AutoRAG.egg-info/SOURCES.txt

@@ -104,6 +104,7 @@ autorag/nodes/promptmaker/base.py
 autorag/nodes/promptmaker/fstring.py
 autorag/nodes/promptmaker/long_context_reorder.py
 autorag/nodes/promptmaker/run.py
+autorag/nodes/promptmaker/window_replacement.py
 autorag/nodes/queryexpansion/__init__.py
 autorag/nodes/queryexpansion/base.py
 autorag/nodes/queryexpansion/hyde.py
@@ -227,6 +228,7 @@ docs/source/nodes/passage_reranker/upr.md
 docs/source/nodes/prompt_maker/fstring.md
 docs/source/nodes/prompt_maker/long_context_reorder.md
 docs/source/nodes/prompt_maker/prompt_maker.md
+docs/source/nodes/prompt_maker/window_replacement.md
 docs/source/nodes/query_expansion/hyde.md
 docs/source/nodes/query_expansion/multi_query_expansion.md
 docs/source/nodes/query_expansion/query_decompose.md
@@ -250,6 +252,7 @@ sample_config/config_korean.yaml
 sample_config/extracted_sample.yaml
 sample_config/full.yaml
 sample_config/simple_local.yaml
+sample_config/simple_ollama.yaml
 sample_config/simple_openai.yaml
 sample_dataset/README.md
 sample_dataset/eli5/load_eli5_dataset.py
@@ -323,6 +326,7 @@ tests/autorag/nodes/promptmaker/test_fstring.py
 tests/autorag/nodes/promptmaker/test_long_context_reorder.py
 tests/autorag/nodes/promptmaker/test_prompt_maker_base.py
 tests/autorag/nodes/promptmaker/test_prompt_maker_run.py
+tests/autorag/nodes/promptmaker/test_window_replacement.py
 tests/autorag/nodes/queryexpansion/test_hyde.py
 tests/autorag/nodes/queryexpansion/test_multi_query_expansion.py
 tests/autorag/nodes/queryexpansion/test_pass_query_expansion.py

{autorag-0.2.8 → autorag-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.8
+Version: 0.2.9
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

autorag-0.2.9/autorag/VERSION

@@ -0,0 +1 @@
+0.2.9

{autorag-0.2.8 → autorag-0.2.9}/autorag/data/qacreation/base.py

@@ -3,6 +3,7 @@ import uuid
 from typing import Callable, Optional
 
 import pandas as pd
+from tqdm import tqdm
 
 from autorag.utils.util import save_parquet_safe
 
@@ -15,6 +16,7 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
                            output_filepath: Optional[str] = None,
                            upsert: bool = False,
                            random_state: int = 42,
+                           cache_batch: int = 32,
                            **kwargs) -> pd.DataFrame:
     """
     Make single content (single-hop, single-document) QA dataset using given qa_creation_func.
@@ -32,6 +34,9 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
     :param upsert: If true, the function will overwrite the existing file if it exists.
         Default is False.
     :param random_state: The random state for sampling corpus from the given corpus_df.
+    :param cache_batch: The number of batches to use for caching the generated QA dataset.
+        When the cache_batch size data is generated, the dataset will save to the designated output_filepath.
+        If the cache_batch size is too small, the process time will be longer.
     :param kwargs: The keyword arguments for qa_creation_func.
     :return: QA dataset dataframe.
         You can save this as parquet file to use at AutoRAG.
@@ -44,24 +49,31 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
     sampled_corpus = corpus_df.sample(n=content_size, random_state=random_state)
     sampled_corpus = sampled_corpus.reset_index(drop=True)
 
-    qa = qa_creation_func(contents=sampled_corpus['contents'].tolist(), **kwargs)
-    qa_data = pd.DataFrame({
-        'qid': [str(uuid.uuid4()) for _ in range(len(qa))],
-        'qa': qa,
-        'retrieval_gt': sampled_corpus['doc_id'].tolist(),
-    })
-    qa_data = qa_data.explode('qa', ignore_index=True)
-
     def make_query_generation_gt(row):
         return row['qa']['query'], row['qa']['generation_gt']
 
-    qa_data[['query', 'generation_gt']] = qa_data.apply(make_query_generation_gt, axis=1, result_type='expand')
-    qa_data = qa_data.drop(columns=['qa'])
+    qa_data = pd.DataFrame()
+    for idx, i in tqdm(enumerate(range(0, len(sampled_corpus), cache_batch))):
+        qa = qa_creation_func(contents=sampled_corpus['contents'].tolist()[i:i + cache_batch], **kwargs)
+
+        temp_qa_data = pd.DataFrame({
+            'qa': qa,
+            'retrieval_gt': sampled_corpus['doc_id'].tolist()[i:i + cache_batch],
+        })
+        temp_qa_data = temp_qa_data.explode('qa', ignore_index=True)
+        temp_qa_data['qid'] = [str(uuid.uuid4()) for _ in range(len(temp_qa_data))]
+        temp_qa_data[['query', 'generation_gt']] = temp_qa_data.apply(make_query_generation_gt, axis=1,
+                                                                      result_type='expand')
+        temp_qa_data = temp_qa_data.drop(columns=['qa'])
 
-    qa_data['retrieval_gt'] = qa_data['retrieval_gt'].apply(lambda x: [[x]])
-    qa_data['generation_gt'] = qa_data['generation_gt'].apply(lambda x: [x])
+        temp_qa_data['retrieval_gt'] = temp_qa_data['retrieval_gt'].apply(lambda x: [[x]])
+        temp_qa_data['generation_gt'] = temp_qa_data['generation_gt'].apply(lambda x: [x])
 
-    if output_filepath is not None:
-        save_parquet_safe(qa_data, output_filepath, upsert=upsert)
+        if idx == 0:
+            qa_data = temp_qa_data
+        else:
+            qa_data = pd.concat([qa_data, temp_qa_data], ignore_index=True)
+        if output_filepath is not None:
+            save_parquet_safe(qa_data, output_filepath, upsert=upsert)
 
     return qa_data

{autorag-0.2.8 → autorag-0.2.9}/autorag/evaluation/metric/generation.py

@@ -6,7 +6,7 @@ from typing import List, Optional
 
 import evaluate
 import pandas as pd
-import sacrebleu
+from sacrebleu.metrics.bleu import BLEU
 import torch
 from llama_index.core.embeddings import BaseEmbedding
 from llama_index.embeddings.openai import OpenAIEmbedding
@@ -64,12 +64,25 @@ def huggingface_evaluate(instance, key: str,
     return result
 
 
-@generation_metric
-def bleu(gt: List[str], pred: str, **kwargs) -> float:
+@convert_inputs_to_list
+def bleu(generation_gt: List[List[str]], generations: [str], tokenize: str|None = None, smooth_method: str = 'exp', smooth_value: Optional[float] = None, max_ngram_order: int = 4, trg_lang: str = '', **kwargs) -> List[float]:
     """
-    Compute bleu score for generation.
+    Computes the BLEU metric given pred and ground-truth.
+
+    :param tokenize: The tokenizer to use. If None, defaults to language-specific tokenizers with '13a' as the fallback default. check #https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/metrics/bleu.py
+    :param smooth_method: The smoothing method to use ('floor', 'add-k', 'exp' or 'none').
+    :param smooth_value: The smoothing value for `floor` and `add-k` methods. `None` falls back to default value.
+    :param max_ngram_order: If given, it overrides the maximum n-gram order (default: 4) when computing precisions.
+    :param trg_lang: An optional language code to raise potential tokenizer warnings.
+    :param generation_gt: A list of ground truth.
+        Must be 2-d list of string.
+        Because it can be a multiple ground truth.
+    :param generations: A list of generations that LLM generated.
     """
-    return sacrebleu.sentence_bleu(pred, gt, **kwargs).score
+    bleu = BLEU(tokenize=tokenize, smooth_method=smooth_method, smooth_value=smooth_value, max_ngram_order=max_ngram_order, trg_lang=trg_lang, **kwargs)
+
+    result = list(map(lambda x: bleu.sentence_score(x[0], x[1]).score, zip(generations, generation_gt)))
+    return result
 
 
 @convert_inputs_to_list

{autorag-0.2.8 → autorag-0.2.9}/autorag/nodes/promptmaker/__init__.py

@@ -1,2 +1,3 @@
 from .fstring import fstring
 from .long_context_reorder import long_context_reorder
+from .window_replacement import window_replacement

{autorag-0.2.8 → autorag-0.2.9}/autorag/nodes/promptmaker/base.py

@@ -1,11 +1,12 @@
 import functools
 import logging
+import os
 from pathlib import Path
 from typing import List, Union
 
 import pandas as pd
 
-from autorag.utils import result_to_dataframe
+from autorag.utils import result_to_dataframe, fetch_contents
 
 logger = logging.getLogger("AutoRAG")
 
@@ -31,6 +32,14 @@ def prompt_maker_node(func):
             assert "retrieve_scores" in previous_result.columns, "previous_result must have retrieve_scores column."
             retrieve_scores = previous_result["retrieve_scores"].tolist()
             return func(prompt, query, retrieved_contents, retrieve_scores)
+        elif func.__name__ == 'window_replacement':
+            retrieved_ids = previous_result["retrieved_ids"].tolist()
+            # load corpus
+            data_dir = os.path.join(project_dir, "data")
+            corpus_data = pd.read_parquet(os.path.join(data_dir, "corpus.parquet"), engine='pyarrow')
+            # get metadata from corpus
+            retrieved_metadata = fetch_contents(corpus_data, retrieved_ids, column_name='metadata')
+            return func(prompt, query, retrieved_contents, retrieved_metadata)
         else:
             raise NotImplementedError(f"Module {func.__name__} is not implemented or not supported.")
 

autorag-0.2.9/autorag/nodes/promptmaker/window_replacement.py

@@ -0,0 +1,46 @@
+import logging
+from typing import List, Dict
+
+from autorag.nodes.promptmaker.base import prompt_maker_node
+
+logger = logging.getLogger("AutoRAG")
+
+
+@prompt_maker_node
+def window_replacement(prompt: str, queries: List[str],
+                       retrieved_contents: List[List[str]],
+                       retrieved_metadata: List[List[Dict]]) -> List[str]:
+    """
+    Replace retrieved_contents with window to create a Prompt
+    (only available for corpus chunked with Sentence window method)
+    You must type a prompt or prompt list at config yaml file like this:
+
+    .. Code:: yaml
+    nodes:
+    - node_type: prompt_maker
+      modules:
+      - module_type: window_replacement
+        prompt: [Answer this question: {query} \n\n {retrieved_contents},
+        Read the passages carefully and answer this question: {query} \n\n Passages: {retrieved_contents}]
+
+    :param prompt: A prompt string.
+    :param queries: List of query strings.
+    :param retrieved_contents: List of retrieved contents.
+    :param retrieved_metadata: List of retrieved metadata.
+    :return: Prompts that made by window_replacement.
+    """
+
+    def window_replacement_row(_prompt: str, _query: str, _retrieved_contents, _retrieved_metadata: List[Dict]) -> str:
+        window_list = []
+        for content, metadata in zip(_retrieved_contents, _retrieved_metadata):
+            if 'window' in metadata:
+                window_list.append(metadata['window'])
+            else:
+                window_list.append(content)
+                logger.info("Only available for corpus chunked with Sentence window method."
+                            "window_replacement will not proceed.")
+        contents_str = "\n\n".join(window_list)
+        return _prompt.format(query=_query, retrieved_contents=contents_str)
+
+    return list(map(lambda x: window_replacement_row(prompt, x[0], x[1], x[2]),
+                    zip(queries, retrieved_contents, retrieved_metadata)))

{autorag-0.2.8 → autorag-0.2.9}/autorag/support.py

@@ -58,6 +58,7 @@ def get_support_modules(module_name: str) -> Callable:
         # prompt_maker
         'fstring': ('autorag.nodes.promptmaker', 'fstring'),
         'long_context_reorder': ('autorag.nodes.promptmaker', 'long_context_reorder'),
+        'window_replacement': ('autorag.nodes.promptmaker', 'window_replacement'),
         # generator
         'llama_index_llm': ('autorag.nodes.generator', 'llama_index_llm'),
         'vllm': ('autorag.nodes.generator', 'vllm'),

{autorag-0.2.8 → autorag-0.2.9}/docs/source/api_spec/autorag.nodes.promptmaker.rst

@@ -36,6 +36,14 @@ autorag.nodes.promptmaker.run module
    :undoc-members:
    :show-inheritance:
 
+autorag.nodes.promptmaker.window\_replacement module
+----------------------------------------------------
+
+.. automodule:: autorag.nodes.promptmaker.window_replacement
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

{autorag-0.2.8 → autorag-0.2.9}/docs/source/data_creation/tutorial.md

@@ -80,7 +80,7 @@ from autorag.data.qacreation import generate_qa_llama_index, make_single_content
 corpus_df = pd.read_parquet('path/to/corpus.parquet')
 llm = OpenAI(model='gpt-3.5-turbo', temperature=1.0)
 qa_df = make_single_content_qa(corpus_df, 50, generate_qa_llama_index, llm=llm, question_num_per_content=1,
-                               output_filepath='path/to/qa.parquet')
+                               output_filepath='path/to/qa.parquet', cache_batch=64)
 ```
 
 `generate_qa_llama_index` is a function designed to generate **questions** and its **generation_gt** per content.
@@ -94,6 +94,15 @@ Which means it uses only one passage per question for answering the question.
 Passage is chunked units from raw data.
 ```
 
+```{admonition} Auto-save feature
+From AutoRAG v0.2.9, the auto-save feature added! 
+Now, you don't have to afraid that something wrong while the data generation.
+The data will save automatically to the input `output_filepath`.
+
+You can set how often you want to save the result to the file. 
+Just adjust `cache_batch` parameter. Default is 32.
+```
+
 ## Use custom prompt
 
 You can use custom prompt to generate qa data.

{autorag-0.2.8 → autorag-0.2.9}/docs/source/local_model.md

@@ -25,10 +25,14 @@ myst:
 
 Most of the modules that using LLM model can take `llm` parameter to specify the LLM model.
 
+- [llama_index_llm](nodes/generator/llama_index_llm.md)
+
+The following modules can use generator module, which including `llama_index_llm`.
+
 - [hyde](nodes/query_expansion/hyde.md)
 - [query_decompose](nodes/query_expansion/query_decompose.md)
+- [multi_query_expansion](nodes/query_expansion/multi_query_expansion.md)
 - [tree_summarize](nodes/passage_compressor/tree_summarize.md)
-- [llama_index_llm](nodes/generator/llama_index_llm.md)
 
 ### Supporting LLM models
 
@@ -114,7 +118,8 @@ To change the embedding model, you can change the `embedding_model` parameter to
 |               [cointegrated/rubert-tiny2](https://huggingface.co/cointegrated/rubert-tiny2)               | huggingface_cointegrated_rubert_tiny2 |
 | [sentence-transformers/all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) |     huggingface_all_mpnet_base_v2     |
 
-For example, if you want to use OpenAI curie embedding model, you can set `embedding_model` parameter to `openai_curie`.
+For example, if you want to use OpenAI text embedding large model, you can set `embedding_model` parameter
+to `openai_embed_3_large`.
 
 ```yaml
 nodes:
@@ -126,11 +131,6 @@ nodes:
             embedding_model: openai
 ```
 
-```{attention}
-You can't pass embedding model parameters at the config yaml file like LLM models.
-Because the embedding model is initialized at the beginning of the AutoRAG program.
-```
-
 ### Add your embedding models
 
 You can add more embedding models for AutoRAG.
@@ -141,13 +141,19 @@ execute the following code.
 
 ```python
 import autorag
+from autorag import LazyInit
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
-autorag.generator_models['kosimcse'] = HuggingFaceEmbedding("BM-K/KoSimCSE-roberta-multitask")
+autorag.embedding_models['kosimcse'] = LazyInit(HuggingFaceEmbedding, model_name="BM-K/KoSimCSE-roberta-multitask")
 ```
 
 Then you can use `kosimcse` at config yaml file.
 
 ```{caution}
-When you add new LLM model, you should add instance of the `BaseEmbedding` class from LlamaIndex.
+When you add new embedding model, you should use `LazyInit` class from autorag. The additional parameters have to be keyword parameter in the `LazyInit` initialization. 
 ```
+
+## Use vllm
+
+You can use vllm to use local LLM. For more information, please check out [vllm](nodes/generator/vllm.md) generator
+module docs.

{autorag-0.2.8 → autorag-0.2.9}/docs/source/nodes/passage_reranker/koreranker.md

@@ -12,7 +12,11 @@ More details can be found [here](https://huggingface.co/Dongjin-kr/ko-reranker).
 
 
 ## **Module Parameters**
-- **Not Applicable (N/A):** There are no direct module parameters specified for the `ko-reranker` module.
+
+(Optional) `batch`
+
+- Specify batch size of the query to the Ko-reranker model.
+- default is 64.
 
 ## **Example config.yaml**
 ```yaml

{autorag-0.2.8 → autorag-0.2.9}/docs/source/nodes/passage_reranker/monot5.md

@@ -17,6 +17,10 @@ given query.
       allowing for flexible integration of different monoT5 variants.
     - default is `castorini/monot5-3b-msmarco-10k`
 
+- (Optional) `batch`
+    - Specify batch size of the query to the TART model.
+    - default is 64.
+
 ## **Example config.yaml**
 
 ```yaml

{autorag-0.2.8 → autorag-0.2.9}/docs/source/nodes/passage_reranker/tart.md

@@ -14,6 +14,11 @@ The `TART` module is a reranker based on [TART](https://arxiv.org/pdf/2211.09260
 - Specifies instructions for the reranking process. 
 - default is `Find passage to answer given question`
 
+(Optional) `batch`
+
+- Specify batch size of the query to the TART model.
+- default is 64.
+
 ## **Example config.yaml**
 ```yaml
 modules:

{autorag-0.2.8 → autorag-0.2.9}/docs/source/nodes/prompt_maker/prompt_maker.md

@@ -68,4 +68,5 @@ maxdepth: 1
 ---
 fstring.md
 long_context_reorder.md
+window_replacement.md
 ```

autorag-0.2.9/docs/source/nodes/prompt_maker/window_replacement.md

@@ -0,0 +1,33 @@
+---
+myst:
+  html_meta:
+    title: AutoRAG - Window Replacement
+    description: Learn about Window Replacement module in AutoRAG
+    keywords: AutoRAG,RAG,Advanced RAG,prompt
+---
+
+# Window Replacement
+
+📌Only available for corpus chunked with `sentence window` method
+
+The `window_replacement` module is prompt maker based on based
+on [llama_index](https://docs.llamaindex.ai/en/stable/examples/node_postprocessor/MetadataReplacementDemo/).
+
+Replace retrieved_contents with window to create a Prompt. This is most useful for large documents/indexes, as it helps
+to retrieve more fine-grained details.
+
+Make a prompt using `window_replacement` from a query and retrieved_contents.
+
+## **Module Parameters**
+
+**prompt**: This is the prompt that will be input to llm. Since it is created using an fstring, it must
+contain `{query}` and `{retreived_contents}`.
+
+## **Example config.yaml**
+
+```yaml
+modules:
+  - module_type: window_replacement
+    prompt: [ "Tell me something about the question: {query} \n\n {retrieved_contents}",
+              "Question: {query} \n Something to read: {retrieved_contents} \n What's your answer?" ]
+```

{autorag-0.2.8 → autorag-0.2.9}/sample_config/config_korean.yaml

@@ -7,19 +7,16 @@ node_lines:
       embedding_model: openai
     - module_type: bm25
       bm25_tokenizer: ko_kiwi
-    top_k: 3
+    top_k: 20
     strategy:
-      metrics:
-      - retrieval_f1
-      - retrieval_recall
-      - retrieval_precision
+      metrics: [ retrieval_recall, retrieval_precision, retrieval_map ]
   - node_type: passage_reranker
     modules:
     - module_type: koreranker
     - module_type: pass_reranker
     - module_type: cohere_reranker
     strategy:
-      metrics: [ retrieval_f1, retrieval_recall, retrieval_precision ]
+      metrics: [ retrieval_recall, retrieval_precision, retrieval_map ]
     top_k: 3
 - node_line_name: post_retrieve_node_line
   nodes:
@@ -30,21 +27,22 @@ node_lines:
     strategy:
       generator_modules:
       - batch: 2
-        llm: openai
-        module_type: llama_index_llm
+        module_type: openai_llm
       metrics:
-      - bleu
-      - meteor
-      - rouge
+        - metric_name: rouge
+        - embedding_model: openai
+          metric_name: sem_score
+        - metric_name: bert_score
+          lang: ko
   - modules:
     - batch: 2
-      llm: openai
-      model: gpt-4
-      module_type: llama_index_llm
+      llm: gpt-4o
+      module_type: openai_llm
     node_type: generator
     strategy:
       metrics:
-      - metric_name: bleu
-      - metric_name: meteor
+      - metric_name: rouge
       - embedding_model: openai
         metric_name: sem_score
+      - metric_name: bert_score
+        lang: ko

{autorag-0.2.8 → autorag-0.2.9}/sample_config/full.yaml

@@ -140,6 +140,9 @@ node_lines:
         - module_type: long_context_reorder
           prompt: [ "Tell me something about the question: {query} \n\n {retrieved_contents}",
                     "Question: {query} \n Something to read: {retrieved_contents} \n What's your answer?" ]
+        - module_type: window_replacement
+          prompt: [ "Tell me something about the question: {query} \n\n {retrieved_contents}",
+                    "Question: {query} \n Something to read: {retrieved_contents} \n What's your answer?" ]
     - node_type: generator
       strategy:
         metrics:

{autorag-0.2.8 → autorag-0.2.9}/sample_config/simple_local.yaml

@@ -13,13 +13,13 @@ node_lines:
   nodes:
     - node_type: prompt_maker
       strategy:
-        metrics: [bleu, meteor, rouge]
+        metrics: [ meteor, rouge, bert_score ]
       modules:
         - module_type: fstring
           prompt: "Read the passages and answer the given question. \n Question: {query} \n Passage: {retrieved_contents} \n Answer : "
     - node_type: generator
       strategy:
-        metrics: [bleu, meteor, rouge]
+        metrics: [ meteor, rouge, bert_score ]
       modules:
         - module_type: vllm
           llm: mistralai/Mistral-7B-Instruct-v0.2

autorag-0.2.9/sample_config/simple_ollama.yaml

@@ -0,0 +1,49 @@
+node_lines:
+  - node_line_name: retrieve_node_line
+    nodes:
+      - node_type: retrieval
+        strategy:
+          metrics: [ retrieval_f1, retrieval_recall, retrieval_precision ]
+        top_k: 3
+        modules:
+          - module_type: bm25
+          - module_type: vectordb
+            embedding_model: huggingface_all_mpnet_base_v2
+          - module_type: hybrid_rrf
+            target_modules: ('bm25', 'vectordb')
+            rrf_k: [ 3, 5, 10 ]
+          - module_type: hybrid_cc
+            target_modules: ('bm25', 'vectordb')
+            weights:
+              - (0.5, 0.5)
+              - (0.3, 0.7)
+              - (0.7, 0.3)
+          - module_type: hybrid_rsf
+            target_modules: ('bm25', 'vectordb')
+            weights:
+              - (0.5, 0.5)
+              - (0.3, 0.7)
+              - (0.7, 0.3)
+          - module_type: hybrid_dbsf
+            target_modules: ('bm25', 'vectordb')
+            weights:
+              - (0.5, 0.5)
+              - (0.3, 0.7)
+              - (0.7, 0.3)
+  - node_line_name: post_retrieve_node_line
+    nodes:
+      - node_type: prompt_maker
+        strategy:
+          metrics: [ meteor, rouge, bert_score ]
+        modules:
+          - module_type: fstring
+            prompt: "Read the passages and answer the given question. \n Question: {query} \n Passage: {retrieved_contents} \n Answer : "
+      - node_type: generator
+        strategy:
+          metrics: [ meteor, rouge, bert_score ]
+        modules:
+          - module_type: llama_index_llm
+            llm: ollama
+            model: llama3
+            temperature: [ 0.1, 0.5, 1.0 ]
+            batch: 1

{autorag-0.2.8 → autorag-0.2.9}/tests/autorag/data/qacreation/test_base_qacreation.py

@@ -48,8 +48,34 @@ def test_single_content_qa(qa_parquet_filepath):
         upsert=True,
     )
     validate_qa_dataset(qa_df)
+    assert len(qa_df) == qa_df['qid'].nunique()
     assert len(qa_df) == 6
     assert qa_df['retrieval_gt'].tolist()[0] == qa_df['retrieval_gt'].tolist()[1]
 
     assert all([len(x) == 1 and len(x[0]) == 1 for x in qa_df['retrieval_gt'].tolist()])
     assert all([len(x) == 1 for x in qa_df['generation_gt'].tolist()])
+
+
+@patch.object(
+    MockLLM,
+    "acomplete",
+    acomplete_qa_creation,
+)
+def test_single_content_qa_long_cache_batch(qa_parquet_filepath):
+    corpus_df = pd.read_parquet(os.path.join(resource_dir, "corpus_data_sample.parquet"))
+    qa_df = make_single_content_qa(
+        corpus_df,
+        content_size=30,
+        qa_creation_func=generate_qa_llama_index,
+        output_filepath=qa_parquet_filepath,
+        llm=MockLLM(),
+        question_num_per_content=2,
+        upsert=True,
+        cache_batch=2,
+    )
+    validate_qa_dataset(qa_df)
+    assert len(qa_df) == 60
+    assert qa_df['retrieval_gt'].tolist()[0] == qa_df['retrieval_gt'].tolist()[1]
+
+    assert all([len(x) == 1 and len(x[0]) == 1 for x in qa_df['retrieval_gt'].tolist()])
+    assert all([len(x) == 1 for x in qa_df['generation_gt'].tolist()])

{autorag-0.2.8 → autorag-0.2.9}/tests/autorag/evaluate/metric/test_generation_metric.py

@@ -47,7 +47,6 @@ def ko_base_test_generation_metrics(func, solution, **kwargs):
     assert all(list(map(lambda x: x[0] == pytest.approx(x[1], 0.001),
                         zip(scores, solution))))
 
-
 def test_bleu():
     base_test_generation_metrics(bleu, [51.1507, 23.5783, 100.0], lowercase=True)
 

autorag-0.2.9/tests/autorag/nodes/promptmaker/test_prompt_maker_base.py

@@ -0,0 +1,28 @@
+import pandas as pd
+
+prompt = "Answer this question: {query} \n\n {retrieved_contents}"
+queries = ["What is the capital of Japan?", "What is the capital of China?"]
+retrieved_contents = [
+    ["Tokyo is the capital of Japan.", "Tokyo, the capital of Japan, is a huge metropolitan city."],
+    ["Beijing is the capital of China.", "Beijing, the capital of China, is a huge metropolitan city."]]
+retrieve_scores = [[0.9, 0.8], [0.9, 0.8]]
+retrieved_ids = [["doc1", "doc2"], ["doc3", "doc4"]]
+previous_result = pd.DataFrame({
+    "query": queries,
+    "retrieved_contents": retrieved_contents,
+    "retrieve_scores": retrieve_scores,
+    "retrieved_ids": retrieved_ids
+})
+
+doc_id = ["doc1", "doc2", "doc3", "doc4", "doc5"]
+contents = ["This is a test document 1.", "This is a test document 2.", "This is a test document 3.",
+            "This is a test document 4.", "This is a test document 5."]
+metadata = [{'window': 'havertz arsenal doosan minji naeun gaeun lets go'} for _ in range(5)]
+corpus_df = pd.DataFrame({"doc_id": doc_id, "contents": contents, "metadata": metadata})
+
+retrieved_metadata = [
+    [{'window': 'havertz arsenal doosan minji naeun gaeun lets go'},
+     {'window': 'havertz arsenal doosan minji naeun gaeun lets go'}],
+    [{'window': 'havertz arsenal doosan minji naeun gaeun lets go'},
+     {'window': 'havertz arsenal doosan minji naeun gaeun lets go'}]
+]