AutoRAG 0.2.12__tar.gz → 0.2.14__tar.gz

autorag-0.2.14/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG] "
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Full Error log**
+If applicable, add full error log to help explain your problem.
+
+**Code that bug is happened**
+If applicable, add the code that bug is happened.
+(Especially, your AutoRAG YAML file or python codes that you wrote)
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Windows, Linux, MacOS]
+ - Python version [e.g. 3.10]
+
+**Additional context**
+Add any other context about the problem here.

autorag-0.2.14/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[Feature Request]"
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

{autorag-0.2.12 → autorag-0.2.14}/.github/workflows/test.yml

@@ -16,6 +16,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '17'
       - name: Upgrade pip
         run: |
           python3 -m pip install --upgrade pip
@@ -24,7 +28,7 @@ jobs:
           sudo apt-get install gcc
       - name: Install AutoRAG
         run: |
-          pip install -e .
+          pip install -e '.[all]'
       - name: Install dependencies
         run: |
           pip install -r tests/requirements.txt

{autorag-0.2.12 → autorag-0.2.14}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.12
+Version: 0.2.14
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -221,13 +221,12 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy<2.0.0
-Requires-Dist: pandas
+Requires-Dist: pandas>=2.1.0
 Requires-Dist: tqdm
 Requires-Dist: tiktoken>=0.7.0
 Requires-Dist: openai>=1.0.0
 Requires-Dist: rank_bm25
 Requires-Dist: transformers
-Requires-Dist: swifter
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: fastparquet
@@ -242,15 +241,15 @@ Requires-Dist: uvicorn
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
-Requires-Dist: cohere<=5.6.2
+Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score
 Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
 Requires-Dist: ragas
-Requires-Dist: kiwipiepy
 Requires-Dist: llmlingua
+Requires-Dist: peft
 Requires-Dist: llama-index>=0.10.1
 Requires-Dist: llama-index-core>=0.10.1
 Requires-Dist: llama-index-readers-file
@@ -268,6 +267,11 @@ Requires-Dist: seaborn
 Requires-Dist: ipykernel
 Requires-Dist: ipywidgets
 Requires-Dist: ipywidgets_bokeh
+Provides-Extra: ko
+Requires-Dist: kiwipiepy; extra == "ko"
+Requires-Dist: konlpy; extra == "ko"
+Provides-Extra: all
+Requires-Dist: AutoRAG[ko]; extra == "all"
 
 # AutoRAG
 
@@ -282,7 +286,6 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 ### 💪 Colab Tutorial
 
 - [Step 1: Basic of AutoRAG | Optimizing your RAG pipeline](https://colab.research.google.com/drive/19OEQXO_pHN6gnn2WdfPd4hjnS-4GurVd?usp=sharing)
-- [Step 2: Create evaluation dataset](https://colab.research.google.com/drive/1HXjVHCLTaX7mkmZp3IKlEPt0B3jVeHvP#scrollTo=cgFUCuaUZvTr)
 
 ---
 
@@ -330,7 +333,7 @@ Try now and find the best RAG pipeline for your own use-case.
 
 ## ❗Supporting Nodes & modules
 
-![module_1](https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/828d8752-13f2-4918-8ca4-2605a86596cb)
+![module_1](https://github.com/user-attachments/assets/d49cc7b4-56a6-4b5f-bb3d-2fc5b962996b)
 ![module_2](https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/4db3d83f-3178-4b81-bad1-b751dcd31a13)
 You can check our all supporting Nodes & modules
 at [here](https://edai.notion.site/Supporting-Nodes-modules-0ebc7810649f4e41aead472a92976be4?pvs=4)

{autorag-0.2.12 → autorag-0.2.14}/AutoRAG.egg-info/SOURCES.txt

@@ -7,6 +7,8 @@ pyproject.toml
 requirements.txt
 .github/FUNDING.yml
 .github/dependabot.yml
+.github/ISSUE_TEMPLATE/bug_report.md
+.github/ISSUE_TEMPLATE/feature_request.md
 .github/workflows/publish.yml
 .github/workflows/sphinx.yml
 .github/workflows/test.yml
@@ -25,6 +27,7 @@ autorag/evaluator.py
 autorag/node_line.py
 autorag/strategy.py
 autorag/support.py
+autorag/validator.py
 autorag/web.py
 autorag/data/__init__.py
 autorag/data/corpus/__init__.py
@@ -264,6 +267,7 @@ tests/autorag/test_deploy.py
 tests/autorag/test_evaluator.py
 tests/autorag/test_strategy.py
 tests/autorag/test_support.py
+tests/autorag/test_validator.py
 tests/autorag/test_web.py
 tests/autorag/data/corpus/test_base.py
 tests/autorag/data/corpus/test_langchain.py

{autorag-0.2.12 → autorag-0.2.14}/AutoRAG.egg-info/requires.txt

@@ -1,11 +1,10 @@
 numpy<2.0.0
-pandas
+pandas>=2.1.0
 tqdm
 tiktoken>=0.7.0
 openai>=1.0.0
 rank_bm25
 transformers
-swifter
 pyyaml
 pyarrow
 fastparquet
@@ -20,15 +19,15 @@ uvicorn
 torch
 sentencepiece
 guidance
-cohere<=5.6.2
+cohere>=5.8.0
 tokenlog>=0.0.2
 aiohttp
 bert_score
 sentence-transformers
 FlagEmbedding
 ragas
-kiwipiepy
 llmlingua
+peft
 llama-index>=0.10.1
 llama-index-core>=0.10.1
 llama-index-readers-file
@@ -46,3 +45,10 @@ seaborn
 ipykernel
 ipywidgets
 ipywidgets_bokeh
+
+[all]
+AutoRAG[ko]
+
+[ko]
+kiwipiepy
+konlpy

{autorag-0.2.12 → autorag-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.12
+Version: 0.2.14
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -221,13 +221,12 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy<2.0.0
-Requires-Dist: pandas
+Requires-Dist: pandas>=2.1.0
 Requires-Dist: tqdm
 Requires-Dist: tiktoken>=0.7.0
 Requires-Dist: openai>=1.0.0
 Requires-Dist: rank_bm25
 Requires-Dist: transformers
-Requires-Dist: swifter
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: fastparquet
@@ -242,15 +241,15 @@ Requires-Dist: uvicorn
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
-Requires-Dist: cohere<=5.6.2
+Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score
 Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
 Requires-Dist: ragas
-Requires-Dist: kiwipiepy
 Requires-Dist: llmlingua
+Requires-Dist: peft
 Requires-Dist: llama-index>=0.10.1
 Requires-Dist: llama-index-core>=0.10.1
 Requires-Dist: llama-index-readers-file
@@ -268,6 +267,11 @@ Requires-Dist: seaborn
 Requires-Dist: ipykernel
 Requires-Dist: ipywidgets
 Requires-Dist: ipywidgets_bokeh
+Provides-Extra: ko
+Requires-Dist: kiwipiepy; extra == "ko"
+Requires-Dist: konlpy; extra == "ko"
+Provides-Extra: all
+Requires-Dist: AutoRAG[ko]; extra == "all"
 
 # AutoRAG
 
@@ -282,7 +286,6 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 ### 💪 Colab Tutorial
 
 - [Step 1: Basic of AutoRAG | Optimizing your RAG pipeline](https://colab.research.google.com/drive/19OEQXO_pHN6gnn2WdfPd4hjnS-4GurVd?usp=sharing)
-- [Step 2: Create evaluation dataset](https://colab.research.google.com/drive/1HXjVHCLTaX7mkmZp3IKlEPt0B3jVeHvP#scrollTo=cgFUCuaUZvTr)
 
 ---
 
@@ -330,7 +333,7 @@ Try now and find the best RAG pipeline for your own use-case.
 
 ## ❗Supporting Nodes & modules
 
-![module_1](https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/828d8752-13f2-4918-8ca4-2605a86596cb)
+![module_1](https://github.com/user-attachments/assets/d49cc7b4-56a6-4b5f-bb3d-2fc5b962996b)
 ![module_2](https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/4db3d83f-3178-4b81-bad1-b751dcd31a13)
 You can check our all supporting Nodes & modules
 at [here](https://edai.notion.site/Supporting-Nodes-modules-0ebc7810649f4e41aead472a92976be4?pvs=4)

{autorag-0.2.12 → autorag-0.2.14}/README.md

@@ -11,7 +11,6 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 ### 💪 Colab Tutorial
 
 - [Step 1: Basic of AutoRAG | Optimizing your RAG pipeline](https://colab.research.google.com/drive/19OEQXO_pHN6gnn2WdfPd4hjnS-4GurVd?usp=sharing)
-- [Step 2: Create evaluation dataset](https://colab.research.google.com/drive/1HXjVHCLTaX7mkmZp3IKlEPt0B3jVeHvP#scrollTo=cgFUCuaUZvTr)
 
 ---
 
@@ -59,7 +58,7 @@ Try now and find the best RAG pipeline for your own use-case.
 
 ## ❗Supporting Nodes & modules
 
-![module_1](https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/828d8752-13f2-4918-8ca4-2605a86596cb)
+![module_1](https://github.com/user-attachments/assets/d49cc7b4-56a6-4b5f-bb3d-2fc5b962996b)
 ![module_2](https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/4db3d83f-3178-4b81-bad1-b751dcd31a13)
 You can check our all supporting Nodes & modules
 at [here](https://edai.notion.site/Supporting-Nodes-modules-0ebc7810649f4e41aead472a92976be4?pvs=4)

autorag-0.2.14/autorag/VERSION

@@ -0,0 +1 @@
+0.2.14

{autorag-0.2.12 → autorag-0.2.14}/autorag/__init__.py

@@ -9,19 +9,16 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.embeddings.openai import OpenAIEmbedding
 from llama_index.embeddings.openai import OpenAIEmbeddingModelType
 from llama_index.llms.huggingface import HuggingFaceLLM
-from llama_index.llms.openai import OpenAI
 from llama_index.llms.ollama import Ollama
+from llama_index.llms.openai import OpenAI
 from llama_index.llms.openai_like import OpenAILike
 from rich.logging import RichHandler
-from swifter import set_defaults
 
 version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'VERSION')
 
 with open(version_path, 'r') as f:
     __version__ = f.read().strip()
 
-set_defaults(allow_dask_on_strings=True)
-
 
 class LazyInit:
     def __init__(self, factory, *args, **kwargs):
@@ -50,7 +47,8 @@ embedding_models = {
     'huggingface_cointegrated_rubert_tiny2': LazyInit(HuggingFaceEmbedding, model_name="cointegrated/rubert-tiny2"),
     'huggingface_all_mpnet_base_v2': LazyInit(HuggingFaceEmbedding,
                                               model_name="sentence-transformers/all-mpnet-base-v2",
-                                              max_length=512, )
+                                              max_length=512, ),
+    'huggingface_bge_m3': LazyInit(HuggingFaceEmbedding, model_name="BAAI/bge-m3"),
 }
 
 generator_models = {

{autorag-0.2.12 → autorag-0.2.14}/autorag/cli.py

@@ -12,6 +12,7 @@ from autorag import dashboard
 from autorag.deploy import Runner
 from autorag.deploy import extract_best_config as original_extract_best_config
 from autorag.evaluator import Evaluator
+from autorag.validator import Validator
 
 logger = logging.getLogger("AutoRAG")
 
@@ -33,7 +34,6 @@ def evaluate(config, qa_data_path, corpus_data_path, project_dir):
         raise ValueError(f"Config file {config} does not exist.")
     evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir=project_dir)
     evaluator.start_trial(config)
-    logger.info('Evaluation complete.')
 
 
 @click.command()
@@ -95,7 +95,19 @@ def restart_evaluate(trial_path):
     corpus_data_path = os.path.join(project_dir, 'data', 'corpus.parquet')
     evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir)
     evaluator.restart_trial(trial_path)
-    logger.info('Evaluation complete.')
+
+
+@click.command()
+@click.option('--config', '-c', help='Path to config yaml file. Must be yaml or yml file.', type=str)
+@click.option('--qa_data_path', help='Path to QA dataset. Must be parquet file.', type=str)
+@click.option('--corpus_data_path', help='Path to corpus dataset. Must be parquet file.', type=str)
+def validate(config, qa_data_path, corpus_data_path):
+    if not config.endswith('.yaml') and not config.endswith('.yml'):
+        raise ValueError(f"Config file {config} is not a parquet file.")
+    if not os.path.exists(config):
+        raise ValueError(f"Config file {config} does not exist.")
+    validator = Validator(qa_data_path=qa_data_path, corpus_data_path=corpus_data_path)
+    validator.validate(config)
 
 
 cli.add_command(evaluate, 'evaluate')
@@ -104,6 +116,7 @@ cli.add_command(run_web, 'run_web')
 cli.add_command(run_dashboard, 'dashboard')
 cli.add_command(extract_best_config, 'extract_best_config')
 cli.add_command(restart_evaluate, 'restart_evaluate')
+cli.add_command(validate, 'validate')
 
 if __name__ == '__main__':
     cli()

autorag-0.2.14/autorag/data/qacreation/__init__.py

@@ -0,0 +1,2 @@
+from .base import make_single_content_qa, make_qa_with_existing_queries
+from .llama_index import generate_qa_llama_index, generate_answers, generate_qa_llama_index_by_ratio

autorag-0.2.14/autorag/data/qacreation/base.py

@@ -0,0 +1,168 @@
+import logging
+import uuid
+from typing import Callable, Optional, List
+
+import chromadb
+import pandas as pd
+from tqdm import tqdm
+
+import autorag
+from autorag.nodes.retrieval.vectordb import vectordb_ingest, vectordb
+from autorag.utils.util import save_parquet_safe, fetch_contents
+
+logger = logging.getLogger("AutoRAG")
+
+
+def make_single_content_qa(corpus_df: pd.DataFrame,
+                           content_size: int,
+                           qa_creation_func: Callable,
+                           output_filepath: Optional[str] = None,
+                           upsert: bool = False,
+                           random_state: int = 42,
+                           cache_batch: int = 32,
+                           **kwargs) -> pd.DataFrame:
+    """
+    Make single content (single-hop, single-document) QA dataset using given qa_creation_func.
+    It generates a single content QA dataset, which means its retrieval ground truth will be only one.
+    It is the most basic form of QA dataset.
+
+    :param corpus_df: The corpus dataframe to make QA dataset from.
+    :param content_size: This function will generate QA dataset for the given number of contents.
+    :param qa_creation_func: The function to create QA pairs.
+        You can use like `generate_qa_llama_index` or `generate_qa_llama_index_by_ratio`.
+        The input func must have `contents` parameter for the list of content string.
+    :param output_filepath: Optional filepath to save the parquet file.
+        If None, the function will return the processed_data as pd.DataFrame, but do not save as parquet.
+        File directory must exist. File extension must be .parquet
+    :param upsert: If true, the function will overwrite the existing file if it exists.
+        Default is False.
+    :param random_state: The random state for sampling corpus from the given corpus_df.
+    :param cache_batch: The number of batches to use for caching the generated QA dataset.
+        When the cache_batch size data is generated, the dataset will save to the designated output_filepath.
+        If the cache_batch size is too small, the process time will be longer.
+    :param kwargs: The keyword arguments for qa_creation_func.
+    :return: QA dataset dataframe.
+        You can save this as parquet file to use at AutoRAG.
+    """
+    assert content_size > 0, "content_size must be greater than 0."
+    if content_size > len(corpus_df):
+        logger.warning(f"content_size {content_size} is larger than the corpus size {len(corpus_df)}. "
+                       "Setting content_size to the corpus size.")
+        content_size = len(corpus_df)
+    sampled_corpus = corpus_df.sample(n=content_size, random_state=random_state)
+    sampled_corpus = sampled_corpus.reset_index(drop=True)
+
+    def make_query_generation_gt(row):
+        return row['qa']['query'], row['qa']['generation_gt']
+
+    qa_data = pd.DataFrame()
+    for idx, i in tqdm(enumerate(range(0, len(sampled_corpus), cache_batch))):
+        qa = qa_creation_func(contents=sampled_corpus['contents'].tolist()[i:i + cache_batch], **kwargs)
+
+        temp_qa_data = pd.DataFrame({
+            'qa': qa,
+            'retrieval_gt': sampled_corpus['doc_id'].tolist()[i:i + cache_batch],
+        })
+        temp_qa_data = temp_qa_data.explode('qa', ignore_index=True)
+        temp_qa_data['qid'] = [str(uuid.uuid4()) for _ in range(len(temp_qa_data))]
+        temp_qa_data[['query', 'generation_gt']] = temp_qa_data.apply(make_query_generation_gt, axis=1,
+                                                                      result_type='expand')
+        temp_qa_data = temp_qa_data.drop(columns=['qa'])
+
+        temp_qa_data['retrieval_gt'] = temp_qa_data['retrieval_gt'].apply(lambda x: [[x]])
+        temp_qa_data['generation_gt'] = temp_qa_data['generation_gt'].apply(lambda x: [x])
+
+        if idx == 0:
+            qa_data = temp_qa_data
+        else:
+            qa_data = pd.concat([qa_data, temp_qa_data], ignore_index=True)
+        if output_filepath is not None:
+            save_parquet_safe(qa_data, output_filepath, upsert=upsert)
+
+    return qa_data
+
+
+def make_qa_with_existing_queries(
+        corpus_df: pd.DataFrame,
+        existing_query_df: pd.DataFrame,
+        content_size: int,
+        answer_creation_func: Callable,
+        output_filepath: Optional[str] = None,
+        embedding_model: str = 'openai_embed_3_large',
+        collection: Optional[chromadb.Collection] = None,
+        upsert: bool = False,
+        random_state: int = 42,
+        cache_batch: int = 32,
+        top_k: int = 3,
+        **kwargs
+) -> pd.DataFrame:
+    """
+    Make single-hop QA dataset using given qa_creation_func and existing queries.
+    
+    :param corpus_df: The corpus dataframe to make QA dataset from.
+    :param existing_query_df: Dataframe containing existing queries to use for QA pair creation.
+    :param content_size: This function will generate QA dataset for the given number of contents.
+    :param answer_creation_func: The function to create answer with input query.
+    :param output_filepath: Optional filepath to save the parquet file.
+    :param embedding_model: The embedding model to use for vectorization.
+        You can add your own embedding model in the autorag.embedding_models.
+        Please refer to how to add an embedding model in this doc: https://docs.auto-rag.com/local_model.html
+        The default is 'openai_embed_3_large'.
+    :param collection: The chromadb collection to use for vector DB.
+        You can make any chromadb collection and use it here.
+        If you already ingested the corpus_df to the collection, the embedding process will not be repeated.
+        The default is None. If None, it makes a temporary collection.
+    :param upsert: If true, the function will overwrite the existing file if it exists.
+    :param random_state: The random state for sampling corpus from the given corpus_df.
+    :param cache_batch: The number of batches to use for caching the generated QA dataset.
+    :param top_k: The number of sources to refer by model.
+        Default is 3.
+    :param kwargs: The keyword arguments for qa_creation_func.
+    :return: QA dataset dataframe.
+    """
+    assert 'query' in existing_query_df.columns, "existing_query_df must have 'query' column."
+    assert content_size > 0, "content_size must be greater than 0."
+    if content_size > len(corpus_df):
+        logger.warning(f"content_size {content_size} is larger than the corpus size {len(corpus_df)}. "
+                       "Setting content_size to the corpus size.")
+        content_size = len(corpus_df)
+
+    logger.info("Loading local embedding model...")
+    embeddings = autorag.embedding_models[embedding_model]
+
+    # Vector DB creation
+    if collection is None:
+        chroma_client = chromadb.Client()
+        collection_name = "auto-rag"
+        collection = chroma_client.get_or_create_collection(collection_name)
+
+    # embed corpus_df
+    vectordb_ingest(collection, corpus_df, embeddings)
+    vectordb_func = vectordb.__wrapped__
+    retrieved_ids, retrieve_scores = vectordb_func(existing_query_df['query'].tolist(), top_k, collection, embeddings)
+
+    retrieved_contents: List[List[str]] = fetch_contents(corpus_df, retrieved_ids)
+    input_passage_strs: List[str] = list(map(
+        lambda x: '\n'.join([f"Document {i + 1}\n{content}" for i, content in enumerate(x)]),
+        retrieved_contents))
+    retrieved_qa_df = pd.DataFrame({
+        'qid': [str(uuid.uuid4()) for _ in range(len(existing_query_df))],
+        'query': existing_query_df['query'].tolist(),
+        'retrieval_gt': list(map(lambda x: [x], retrieved_ids)),
+        'input_passage_str': input_passage_strs,
+    })
+
+    sample_qa_df = retrieved_qa_df.sample(n=min(content_size, len(retrieved_qa_df)), random_state=random_state)
+
+    generation_gt = answer_creation_func(contents=sample_qa_df['input_passage_str'].tolist(),
+                                         queries=sample_qa_df['query'].tolist(),
+                                         batch=cache_batch,
+                                         **kwargs)
+    qa_df = sample_qa_df.copy(deep=True)
+    qa_df.drop(columns=['input_passage_str'], inplace=True)
+    qa_df['generation_gt'] = generation_gt
+
+    if output_filepath is not None:
+        save_parquet_safe(qa_df, output_filepath, upsert=upsert)
+
+    return qa_df

{autorag-0.2.12 → autorag-0.2.14}/autorag/data/qacreation/llama_index.py

@@ -4,6 +4,7 @@ import random
 from typing import Optional, List, Dict, Any
 
 import pandas as pd
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
 from llama_index.core.service_context_elements.llm_predictor import LLMPredictorType
 
 from autorag.utils.util import process_batch
@@ -53,6 +54,31 @@ def generate_qa_llama_index(
     return results
 
 
+def generate_answers(
+    llm: LLMPredictorType,
+    contents: List[str],
+    queries: List[str],
+    batch: int = 4,
+) -> List[List[Dict]]:
+    """
+    Generate qa sets from the list of contents using existing queries.
+
+    :param llm: Llama index model
+    :param contents: List of content strings.
+    :param queries: List of existing queries.
+    :param batch: The batch size to process asynchronously.
+    :return: 2-d list of dictionaries containing the query and generation_gt.
+    """
+
+    tasks = [
+        generate_basic_answer(llm, content, query)
+        for content, query in zip(contents, queries)
+    ]
+    loops = asyncio.get_event_loop()
+    results = loops.run_until_complete(process_batch(tasks, batch))
+    return results
+
+
 def generate_qa_llama_index_by_ratio(
         llm: LLMPredictorType,
         contents: List[str],
@@ -147,6 +173,21 @@ async def async_qa_gen_llama_index(
     return await generate(content, llm)
 
 
+async def generate_basic_answer(llm: LLMPredictorType, passage_str: str, query: str) -> str:
+    basic_answer_system_prompt = """You are an AI assistant to answer the given question in the provide evidence text.
+    You can find the evidence from the given text about question, and you have to write a proper answer to the given question.
+    You have to preserve the question's language at the answer.
+    For example, if the input question is Korean, the output answer must be in Korean.
+    """
+    user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
+
+    response = await llm.achat(messages=[
+        ChatMessage(role=MessageRole.SYSTEM, content=basic_answer_system_prompt),
+        ChatMessage(role=MessageRole.USER, content=user_prompt)
+    ], temperature=1.0)
+    return response.message.content
+
+
 def validate_llama_index_prompt(prompt: str) -> bool:
     """
     Validate the prompt for the llama index model.

{autorag-0.2.12 → autorag-0.2.14}/autorag/deploy.py

@@ -4,6 +4,7 @@ import uuid
 from copy import deepcopy
 from typing import Optional, Dict, List
 
+import nest_asyncio
 import pandas as pd
 import uvicorn
 import yaml
@@ -65,7 +66,7 @@ def summary_df_to_yaml(summary_df: pd.DataFrame, config_dict: Dict) -> Dict:
     summary_df['categorical_node_line_name'] = pd.Categorical(summary_df['node_line_name'], categories=node_line_names,
                                                               ordered=True)
     summary_df = summary_df.sort_values(by='categorical_node_line_name')
-    grouped = summary_df.groupby('categorical_node_line_name')
+    grouped = summary_df.groupby('categorical_node_line_name', observed=False)
 
     node_lines = [
         {
@@ -207,7 +208,7 @@ class Runner:
 
             {
                 "Query": "your query",
-                "result_column": "answer"
+                "result_column": "generated_texts"
             }
 
         And it returns json response like below:
@@ -222,10 +223,11 @@ class Runner:
         :param port: The port of the api server.
         :param kwargs: Other arguments for uvicorn.run.
         """
+        nest_asyncio.apply()
         logger.info(f"Run api server at {host}:{port}")
-        uvicorn.run(self.app, host=host, port=port, **kwargs)
+        uvicorn.run(self.app, host=host, port=port, loop="asyncio", **kwargs)
 
 
 class RunnerInput(BaseModel):
     query: str
-    result_column: str = "answer"
+    result_column: str = "generated_texts"

{autorag-0.2.12 → autorag-0.2.14}/autorag/evaluation/metric/generation.py

@@ -5,6 +5,7 @@ import os
 from typing import List, Optional
 
 import evaluate
+import nltk
 import pandas as pd
 import torch
 from llama_index.core.embeddings import BaseEmbedding
@@ -109,6 +110,7 @@ def meteor(generation_gt: List[List[str]], generations: List[str],
         Default is 0.5.
     :return: A list of computed metric scores.
     """
+    nltk.download('punkt_tab')
     meteor_instance = evaluate.load("meteor")
     result = huggingface_evaluate(meteor_instance, 'meteor', generation_gt, generations,
                                   alpha=alpha, beta=beta, gamma=gamma)